diff --git a/.changeset/clean-queens-judge.md b/.changeset/clean-queens-judge.md new file mode 100644 index 0000000000..1dd7719c5f --- /dev/null +++ b/.changeset/clean-queens-judge.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +The process of adding or modifying fields in the software-catalog search index has been simplified. For more details, see [how to customize fields in the Software Catalog index](../docs/features/search/how-to-guides.md#how-to-customize-fields-in-the-software-catalog-index). diff --git a/docs/features/search/how-to-guides.md b/docs/features/search/how-to-guides.md index 7267e9fd5b..eb162ce527 100644 --- a/docs/features/search/how-to-guides.md +++ b/docs/features/search/how-to-guides.md @@ -114,6 +114,42 @@ of the `SearchType` component. > Check out the documentation around [integrating search into plugins](../../plugins/integrating-search-into-plugins.md#create-a-collator) for how to create your own collator. +## How to customize fields in the Software Catalog index + +Sometimes you will might want to have ability to control +which data passes to search index in catalog collator, or to customize data for specific kind. +You can easily do that by passing `entityTransformer` callback to `DefaultCatalogCollatorFactory`. +You can either just simply amend default behaviour, or even to write completely new document +(which should follow some required basic structure though). + +> `authorization` and `location` cannot be modified via a `entityTransformer`, `location` can be modified only through `locationTemplate`. + +```diff +// packages/backend/src/plugins/search.ts + +const entityTransformer: CatalogCollatorEntityTransformer = (entity: Entity) => { + if (entity.kind === 'SomeKind') { + return { + // customize here output for 'SomeKind' kind + }; + } + + return { + // and customize default output + ...defaultCatalogCollatorEntityTransformer(entity), + text: 'my super cool text', + }; +}; + +indexBuilder.addCollator({ + collator: DefaultCatalogCollatorFactory.fromConfig(env.config, { + discovery: env.discovery, + tokenManager: env.tokenManager, ++ entityTransformer, + }), +}); +``` + ## How to limit what can be searched in the Software Catalog The Software Catalog includes a wealth of information about the components, diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 6ff9ac4b1f..f15f6b9ba9 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -167,6 +167,11 @@ export class CatalogBuilder { useLegacySingleProcessorValidation(): this; } +// @public (undocumented) +export type CatalogCollatorEntityTransformer = ( + entity: Entity, +) => Omit; + // @alpha export const catalogConditions: Conditions<{ hasAnnotation: PermissionRule< @@ -350,6 +355,9 @@ export class DefaultCatalogCollator { readonly visibilityPermission: Permission; } +// @public (undocumented) +export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer; + // @public (undocumented) export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { // (undocumented) @@ -360,7 +368,7 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { // (undocumented) getCollator(): Promise; // (undocumented) - readonly type: string; + readonly type = 'software-catalog'; // (undocumented) readonly visibilityPermission: Permission; } @@ -373,6 +381,7 @@ export type DefaultCatalogCollatorFactoryOptions = { filter?: GetEntitiesRequest['filter']; batchSize?: number; catalogClient?: CatalogApi; + entityTransformer?: CatalogCollatorEntityTransformer; }; export { DeferredEntity }; diff --git a/plugins/catalog-backend/src/search/util.ts b/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts similarity index 57% rename from plugins/catalog-backend/src/search/util.ts rename to plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts index fd54d397c9..86d71d4c13 100644 --- a/plugins/catalog-backend/src/search/util.ts +++ b/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts @@ -14,16 +14,10 @@ * limitations under the License. */ -import { Entity, isUserEntity, isGroupEntity } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; +import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; -export function getDocumentText(entity: Entity): string { - const documentTexts: string[] = []; - documentTexts.push(entity.metadata.description || ''); - - if (isUserEntity(entity) || isGroupEntity(entity)) { - if (entity.spec?.profile?.displayName) { - documentTexts.push(entity.spec.profile.displayName); - } - } - return documentTexts.join(' : '); -} +/** @public */ +export type CatalogCollatorEntityTransformer = ( + entity: Entity, +) => Omit; diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts index 8b4e5601b4..462e777dff 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts @@ -95,11 +95,6 @@ describe('DefaultCatalogCollatorFactory', () => { ); }); - it('has expected type', () => { - const factory = DefaultCatalogCollatorFactory.fromConfig(config, options); - expect(factory.type).toBe('software-catalog'); - }); - describe('getCollator', () => { let factory: DefaultCatalogCollatorFactory; let collator: Readable; @@ -124,24 +119,28 @@ describe('DefaultCatalogCollatorFactory', () => { const pipeline = TestPipeline.fromCollator(collator); const { documents } = await pipeline.execute(); - expect(documents[0]).toMatchObject({ + expect(documents[0]).toEqual({ title: expectedEntities[0].metadata.name, location: '/catalog/default/component/test-entity', text: expectedEntities[0].metadata.description, namespace: 'default', componentType: expectedEntities[0]!.spec!.type, + kind: expectedEntities[0]!.kind, + type: expectedEntities[0]!.spec!.type, lifecycle: expectedEntities[0]!.spec!.lifecycle, owner: expectedEntities[0]!.spec!.owner, authorization: { resourceRef: 'component:default/test-entity', }, }); - expect(documents[1]).toMatchObject({ + expect(documents[1]).toEqual({ title: expectedEntities[1].metadata.title, location: '/catalog/default/component/test-entity-2', text: expectedEntities[1].metadata.description, namespace: 'default', componentType: expectedEntities[1]!.spec!.type, + kind: expectedEntities[1]!.kind, + type: expectedEntities[1]!.spec!.type, lifecycle: expectedEntities[1]!.spec!.lifecycle, owner: expectedEntities[1]!.spec!.owner, authorization: { @@ -150,6 +149,61 @@ describe('DefaultCatalogCollatorFactory', () => { }); }); + it('maps a returned entity to an expected CatalogEntityDocument with custom transformer', async () => { + const customFactory = DefaultCatalogCollatorFactory.fromConfig(config, { + ...options, + entityTransformer: entity => ({ + title: `custom-title-${ + entity.metadata.title ?? entity.metadata.name + }`, + namespace: 'custom/namespace', + text: 'custom-text', + type: 'custom-type', + componentType: 'custom-component-type', + kind: 'custom-kind', + lifecycle: 'custom-lifecycle', + owner: 'custom-owner', + authorization: { + resourceRef: 'custom:resource/ref', + }, + location: '/custom/location', + }), + }); + const customCollator = await customFactory.getCollator(); + + const pipeline = TestPipeline.fromCollator(customCollator); + const { documents } = await pipeline.execute(); + + expect(documents[0]).toEqual({ + title: 'custom-title-test-entity', + location: '/catalog/default/component/test-entity', + text: 'custom-text', + namespace: 'custom/namespace', + componentType: 'custom-component-type', + kind: 'custom-kind', + type: 'custom-type', + lifecycle: 'custom-lifecycle', + owner: 'custom-owner', + authorization: { + resourceRef: 'component:default/test-entity', + }, + }); + expect(documents[1]).toEqual({ + title: 'custom-title-Test Entity', + location: '/catalog/default/component/test-entity-2', + text: 'custom-text', + namespace: 'custom/namespace', + componentType: 'custom-component-type', + kind: 'custom-kind', + type: 'custom-type', + lifecycle: 'custom-lifecycle', + owner: 'custom-owner', + authorization: { + resourceRef: 'component:default/test-entity-2', + }, + }); + }); + it('maps a returned entity with a custom locationTemplate', async () => { // Provide an alternate location template. factory = DefaultCatalogCollatorFactory.fromConfig(new ConfigReader({}), { diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts index cf082c271c..1c03836e51 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { Config } from '@backstage/config'; import { PluginEndpointDiscovery, TokenManager, @@ -23,8 +24,6 @@ import { CatalogClient, GetEntitiesRequest, } from '@backstage/catalog-client'; -import { stringifyEntityRef } from '@backstage/catalog-model'; -import { Config } from '@backstage/config'; import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; import { catalogEntityReadPermission, @@ -32,7 +31,9 @@ import { } from '@backstage/plugin-catalog-common'; import { Permission } from '@backstage/plugin-permission-common'; import { Readable } from 'stream'; -import { getDocumentText } from './util'; +import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; +import { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer'; +import { stringifyEntityRef } from '@backstage/catalog-model'; /** @public */ export type DefaultCatalogCollatorFactoryOptions = { @@ -42,11 +43,12 @@ export type DefaultCatalogCollatorFactoryOptions = { filter?: GetEntitiesRequest['filter']; batchSize?: number; catalogClient?: CatalogApi; + entityTransformer?: CatalogCollatorEntityTransformer; }; /** @public */ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { - public readonly type: string = 'software-catalog'; + public readonly type = 'software-catalog'; public readonly visibilityPermission: Permission = catalogEntityReadPermission; @@ -55,6 +57,7 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { private batchSize: number; private readonly catalogClient: CatalogApi; private tokenManager: TokenManager; + private entityTransformer: CatalogCollatorEntityTransformer; static fromConfig( _config: Config, @@ -71,6 +74,7 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { filter, catalogClient, tokenManager, + entityTransformer, } = options; this.locationTemplate = @@ -80,23 +84,14 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { this.catalogClient = catalogClient || new CatalogClient({ discoveryApi: discovery }); this.tokenManager = tokenManager; + this.entityTransformer = + entityTransformer ?? defaultCatalogCollatorEntityTransformer; } async getCollator(): Promise { return Readable.from(this.execute()); } - private applyArgsToFormat( - format: string, - args: Record, - ): string { - let formatted = format; - for (const [key, value] of Object.entries(args)) { - formatted = formatted.replace(`:${key}`, value); - } - return formatted.toLowerCase(); - } - private async *execute(): AsyncGenerator { const { token } = await this.tokenManager.getToken(); let entitiesRetrieved = 0; @@ -123,24 +118,30 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { for (const entity of entities) { yield { - title: entity.metadata.title ?? entity.metadata.name, + ...this.entityTransformer(entity), + authorization: { + resourceRef: stringifyEntityRef(entity), + }, location: this.applyArgsToFormat(this.locationTemplate, { namespace: entity.metadata.namespace || 'default', kind: entity.kind, name: entity.metadata.name, }), - text: getDocumentText(entity), - componentType: entity.spec?.type?.toString() || 'other', - type: entity.spec?.type?.toString() || 'other', - namespace: entity.metadata.namespace || 'default', - kind: entity.kind, - lifecycle: (entity.spec?.lifecycle as string) || '', - owner: (entity.spec?.owner as string) || '', - authorization: { - resourceRef: stringifyEntityRef(entity), - }, }; } } } + + private applyArgsToFormat( + format: string, + args: Record, + ): string { + let formatted = format; + + for (const [key, value] of Object.entries(args)) { + formatted = formatted.replace(`:${key}`, value); + } + + return formatted.toLowerCase(); + } } diff --git a/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.test.ts b/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.test.ts new file mode 100644 index 0000000000..8206a58599 --- /dev/null +++ b/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer'; + +const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + title: 'Test Entity', + name: 'test-entity', + description: 'The expected description', + namespace: 'namespace', + }, + spec: { + type: 'some-type', + lifecycle: 'experimental', + owner: 'someone', + }, +}; + +const userEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'test-user-entity', + description: 'The expected user description', + }, + spec: { + profile: { + displayName: 'User 1', + }, + }, +}; + +describe('DefaultCatalogCollatorEntityTransformer', () => { + describe('transform', () => { + it('maps a returned entity', async () => { + const document = defaultCatalogCollatorEntityTransformer(entity); + + expect(document).toMatchObject({ + title: entity.metadata.title, + text: entity.metadata.description, + namespace: entity.metadata.namespace, + componentType: entity.spec.type, + lifecycle: entity.spec.lifecycle, + owner: entity.spec.owner, + }); + }); + + it('maps a returned entity with default fallback', async () => { + const entityWithoutTitle = { + ...entity, + metadata: { + ...entity.metadata, + title: undefined, + namespace: undefined, + }, + spec: { + type: undefined, + lifecycle: undefined, + owner: undefined, + }, + }; + + const document = + defaultCatalogCollatorEntityTransformer(entityWithoutTitle); + + expect(document).toMatchObject({ + title: entity.metadata.name, + text: entity.metadata.description, + namespace: 'default', + componentType: 'other', + lifecycle: '', + owner: '', + }); + }); + + it('maps a returned user entity', async () => { + const document = defaultCatalogCollatorEntityTransformer(userEntity); + + expect(document).toMatchObject({ + title: userEntity.metadata.name, + text: `${userEntity.metadata.description} : ${userEntity.spec.profile.displayName}`, + namespace: 'default', + componentType: 'other', + lifecycle: '', + owner: '', + }); + }); + + it('maps a returned user entity without display name', async () => { + const testEntity = { + ...userEntity, + spec: undefined, + }; + + const document = defaultCatalogCollatorEntityTransformer(testEntity); + + expect(document).toMatchObject({ + title: userEntity.metadata.name, + text: userEntity.metadata.description, + namespace: 'default', + componentType: 'other', + lifecycle: '', + owner: '', + }); + }); + + it('maps a returned group entity', async () => { + const groupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'test-group-entity', + description: 'The expected group description', + }, + spec: { + profile: { + displayName: 'Group 1', + }, + }, + }; + + const document = defaultCatalogCollatorEntityTransformer(groupEntity); + + expect(document).toMatchObject({ + title: groupEntity.metadata.name, + text: `${groupEntity.metadata.description} : ${groupEntity.spec.profile.displayName}`, + namespace: 'default', + componentType: 'other', + lifecycle: '', + owner: '', + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts b/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts new file mode 100644 index 0000000000..2fc0c5d50d --- /dev/null +++ b/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, isGroupEntity, isUserEntity } from '@backstage/catalog-model'; +import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; + +const getDocumentText = (entity: Entity): string => { + const documentTexts: string[] = []; + documentTexts.push(entity.metadata.description || ''); + + if (isUserEntity(entity) || isGroupEntity(entity)) { + if (entity.spec?.profile?.displayName) { + documentTexts.push(entity.spec.profile.displayName); + } + } + + return documentTexts.join(' : '); +}; + +/** @public */ +export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer = + (entity: Entity) => { + return { + title: entity.metadata.title ?? entity.metadata.name, + text: getDocumentText(entity), + componentType: entity.spec?.type?.toString() || 'other', + type: entity.spec?.type?.toString() || 'other', + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: (entity.spec?.owner as string) || '', + }; + }; diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts index 2602f0fada..c2a23e5132 100644 --- a/plugins/catalog-backend/src/search/index.ts +++ b/plugins/catalog-backend/src/search/index.ts @@ -16,6 +16,8 @@ export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory'; +export type { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; +export type { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer'; /** * todo(backstage/techdocs-core): stop exporting this in a future release. diff --git a/plugins/catalog-backend/src/search/util.test.ts b/plugins/catalog-backend/src/search/util.test.ts deleted file mode 100644 index d2d1776338..0000000000 --- a/plugins/catalog-backend/src/search/util.test.ts +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - ComponentEntity, - GroupEntity, - UserEntity, -} from '@backstage/catalog-model'; -import { getDocumentText } from './util'; - -describe('getDocumentText', () => { - describe('kind is not User or Group', () => { - test('contains description if set', () => { - const entity = createComponent(); - entity.metadata.description = 'The expected description'; - const actual = getDocumentText(entity); - expect(actual).toContain(entity.metadata.description); - }); - - test('is empty if description is not set', () => { - const entity = createComponent(); - const actual = getDocumentText(entity); - expect(actual).toEqual(''); - }); - }); - - describe('kind is User', () => { - test('contains display name if set', () => { - const entity = createUser(); - const actual = getDocumentText(entity); - expect(actual).toContain(entity.spec.profile?.displayName); - }); - - test('contains description if set', () => { - const entity = createUser(); - const actual = getDocumentText(entity); - expect(actual).toContain(entity.metadata.description); - }); - - test('contains both description and display name if both are set', () => { - const entity = createUser(); - const actual = getDocumentText(entity); - expect(actual).toContain(entity.spec.profile?.displayName); - expect(actual).toContain(entity.metadata.description); - }); - - test('is empty if description and display name are not set', () => { - const entity = createUser(); - delete entity.metadata.description; - delete entity.spec.profile?.displayName; - const actual = getDocumentText(entity); - expect(actual).toEqual(''); - }); - }); - - describe('kind is Group', () => { - test('contains display name if set', () => { - const entity = createGroup(); - const actual = getDocumentText(entity); - expect(actual).toContain(entity.spec.profile?.displayName); - }); - - test('contains description if set', () => { - const entity = createGroup(); - const actual = getDocumentText(entity); - expect(actual).toContain(entity.metadata.description); - }); - - test('contains both description and display name if both are set', () => { - const entity = createGroup(); - const actual = getDocumentText(entity); - expect(actual).toContain(entity.spec.profile?.displayName); - expect(actual).toContain(entity.metadata.description); - }); - - test('is empty if description and display name are not set', () => { - const entity = createGroup(); - delete entity.metadata.description; - delete entity.spec.profile?.displayName; - const actual = getDocumentText(entity); - expect(actual).toEqual(''); - }); - }); -}); - -function createGroup(): GroupEntity { - return { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'group-1', - description: 'The expected description', - }, - spec: { - type: 'team', - profile: { - displayName: 'Group 1', - }, - children: [], - }, - }; -} - -function createUser(): UserEntity { - return { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'user-1', - description: 'The expected description', - }, - spec: { - profile: { - displayName: 'User 1', - }, - }, - }; -} - -function createComponent(): ComponentEntity { - return { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'component-1', - }, - spec: { - lifecycle: 'experimental', - owner: 'someone', - type: 'service', - }, - }; -}