From 2380506364c6642dddac6032801e144d1f52e4a3 Mon Sep 17 00:00:00 2001 From: Ilya Savich Date: Fri, 9 Dec 2022 15:58:53 +0100 Subject: [PATCH 1/3] Refactor catalog collator, extracted abstract class and entity processor to make it possible to re-use functionality Signed-off-by: Ilya Savich --- .changeset/clean-queens-judge.md | 5 + plugins/catalog-backend/api-report.md | 58 ++++-- ...l.ts => CatalogCollatorEntityProcessor.ts} | 15 +- .../src/search/CatalogCollatorFactory.ts | 132 ++++++++++++ ...aultCatalogCollatorEntityProcessor.test.ts | 193 ++++++++++++++++++ .../DefaultCatalogCollatorEntityProcessor.ts | 78 +++++++ .../search/DefaultCatalogCollatorFactory.ts | 130 ++---------- plugins/catalog-backend/src/search/index.ts | 5 + .../catalog-backend/src/search/util.test.ts | 146 ------------- 9 files changed, 472 insertions(+), 290 deletions(-) create mode 100644 .changeset/clean-queens-judge.md rename plugins/catalog-backend/src/search/{util.ts => CatalogCollatorEntityProcessor.ts} (58%) create mode 100644 plugins/catalog-backend/src/search/CatalogCollatorFactory.ts create mode 100644 plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.test.ts create mode 100644 plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.ts delete mode 100644 plugins/catalog-backend/src/search/util.test.ts diff --git a/.changeset/clean-queens-judge.md b/.changeset/clean-queens-judge.md new file mode 100644 index 0000000000..22f87e6959 --- /dev/null +++ b/.changeset/clean-queens-judge.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Refactored catalog collator, extracted abstract class and entity processor to make it possible to re-use functionality diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index a46abda5d3..8432092796 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -167,6 +167,42 @@ export class CatalogBuilder { useLegacySingleProcessorValidation(): this; } +// @public (undocumented) +export abstract class CatalogCollatorFactory + implements DocumentCollatorFactory +{ + protected constructor(options: CatalogCollatorFactoryOptions); + // (undocumented) + static fromConfig( + _config: Config, + _options: CatalogCollatorFactoryCreateOptions, + ): CatalogCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type: string; + // (undocumented) + readonly visibilityPermission: Permission; +} + +// @public (undocumented) +export type CatalogCollatorFactoryCreateOptions = Omit< + CatalogCollatorFactoryOptions, + 'entityProcessor' | 'type' +>; + +// @public (undocumented) +export type CatalogCollatorFactoryOptions = { + type: string; + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + entityProcessor: CatalogCollatorEntityProcessor; + locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; + batchSize?: number; + catalogClient?: CatalogApi; +}; + // @alpha export const catalogConditions: Conditions<{ hasAnnotation: PermissionRule< @@ -351,29 +387,17 @@ export class DefaultCatalogCollator { } // @public (undocumented) -export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { +export class DefaultCatalogCollatorFactory extends CatalogCollatorFactory { // (undocumented) static fromConfig( _config: Config, options: DefaultCatalogCollatorFactoryOptions, ): DefaultCatalogCollatorFactory; - // (undocumented) - getCollator(): Promise; - // (undocumented) - readonly type: string; - // (undocumented) - readonly visibilityPermission: Permission; } // @public (undocumented) -export type DefaultCatalogCollatorFactoryOptions = { - discovery: PluginEndpointDiscovery; - tokenManager: TokenManager; - locationTemplate?: string; - filter?: GetEntitiesRequest['filter']; - batchSize?: number; - catalogClient?: CatalogApi; -}; +export type DefaultCatalogCollatorFactoryOptions = + CatalogCollatorFactoryCreateOptions; export { DeferredEntity }; @@ -583,4 +607,8 @@ export class UrlReaderProcessor implements CatalogProcessor { cache: CatalogProcessorCache, ): Promise; } + +// Warnings were encountered during analysis: +// +// src/search/CatalogCollatorFactory.d.ts:14:5 - (ae-forgotten-export) The symbol "CatalogCollatorEntityProcessor" needs to be exported by the entry point index.d.ts ``` diff --git a/plugins/catalog-backend/src/search/util.ts b/plugins/catalog-backend/src/search/CatalogCollatorEntityProcessor.ts similarity index 58% rename from plugins/catalog-backend/src/search/util.ts rename to plugins/catalog-backend/src/search/CatalogCollatorEntityProcessor.ts index fd54d397c9..0288cbec24 100644 --- a/plugins/catalog-backend/src/search/util.ts +++ b/plugins/catalog-backend/src/search/CatalogCollatorEntityProcessor.ts @@ -14,16 +14,9 @@ * limitations under the License. */ -import { Entity, isUserEntity, isGroupEntity } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; +import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; -export function getDocumentText(entity: Entity): string { - const documentTexts: string[] = []; - documentTexts.push(entity.metadata.description || ''); - - if (isUserEntity(entity) || isGroupEntity(entity)) { - if (entity.spec?.profile?.displayName) { - documentTexts.push(entity.spec.profile.displayName); - } - } - return documentTexts.join(' : '); +export interface CatalogCollatorEntityProcessor { + process(entity: Entity, locationTemplate: string): CatalogEntityDocument; } diff --git a/plugins/catalog-backend/src/search/CatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/CatalogCollatorFactory.ts new file mode 100644 index 0000000000..8b9a60b264 --- /dev/null +++ b/plugins/catalog-backend/src/search/CatalogCollatorFactory.ts @@ -0,0 +1,132 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { + CatalogApi, + CatalogClient, + GetEntitiesRequest, +} from '@backstage/catalog-client'; +import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; +import { + catalogEntityReadPermission, + CatalogEntityDocument, +} from '@backstage/plugin-catalog-common'; +import { Permission } from '@backstage/plugin-permission-common'; +import { Readable } from 'stream'; +import { CatalogCollatorEntityProcessor } from './CatalogCollatorEntityProcessor'; +import { Config } from '@backstage/config'; + +/** @public */ +export type CatalogCollatorFactoryOptions = { + type: string; + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + entityProcessor: CatalogCollatorEntityProcessor; + locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; + batchSize?: number; + catalogClient?: CatalogApi; +}; + +/** @public */ +export type CatalogCollatorFactoryCreateOptions = Omit< + CatalogCollatorFactoryOptions, + 'entityProcessor' | 'type' +>; + +/** @public */ +export abstract class CatalogCollatorFactory + implements DocumentCollatorFactory +{ + public readonly type: string; + public readonly visibilityPermission: Permission = + catalogEntityReadPermission; + + private locationTemplate: string; + private filter?: GetEntitiesRequest['filter']; + private batchSize: number; + private readonly catalogClient: CatalogApi; + private tokenManager: TokenManager; + private entityProcessor: CatalogCollatorEntityProcessor; + + static fromConfig( + _config: Config, + _options: CatalogCollatorFactoryCreateOptions, + ): CatalogCollatorFactory { + throw new Error('Method should be implemented'); + } + + protected constructor(options: CatalogCollatorFactoryOptions) { + const { + type, + batchSize, + discovery, + locationTemplate, + filter, + catalogClient, + tokenManager, + entityProcessor, + } = options; + + this.type = type; + this.locationTemplate = + locationTemplate || '/catalog/:namespace/:kind/:name'; + this.filter = filter; + this.batchSize = batchSize || 500; + this.catalogClient = + catalogClient || new CatalogClient({ discoveryApi: discovery }); + this.tokenManager = tokenManager; + this.entityProcessor = entityProcessor; + } + + async getCollator(): Promise { + return Readable.from(this.execute()); + } + + private async *execute(): AsyncGenerator { + const { token } = await this.tokenManager.getToken(); + let entitiesRetrieved = 0; + let moreEntitiesToGet = true; + + // Offset/limit pagination is used on the Catalog Client in order to + // limit (and allow some control over) memory used by the search backend + // at index-time. + while (moreEntitiesToGet) { + const entities = ( + await this.catalogClient.getEntities( + { + filter: this.filter, + limit: this.batchSize, + offset: entitiesRetrieved, + }, + { token }, + ) + ).items; + + // Control looping through entity batches. + moreEntitiesToGet = entities.length === this.batchSize; + entitiesRetrieved += entities.length; + + for (const entity of entities) { + yield this.entityProcessor.process(entity, this.locationTemplate); + } + } + } +} diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.test.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.test.ts new file mode 100644 index 0000000000..1183d5be97 --- /dev/null +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.test.ts @@ -0,0 +1,193 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DefaultCatalogCollatorEntityProcessor } from './DefaultCatalogCollatorEntityProcessor'; + +const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + title: 'Test Entity', + name: 'test-entity', + description: 'The expected description', + namespace: 'namespace', + }, + spec: { + type: 'some-type', + lifecycle: 'experimental', + owner: 'someone', + }, +}; + +const userEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'test-user-entity', + description: 'The expected user description', + }, + spec: { + profile: { + displayName: 'User 1', + }, + }, +}; + +const locationTemplate = '/catalog/:namespace/:kind/:name'; + +describe('DefaultCatalogCollatorEntityProcessor', () => { + const entityProcessor = new DefaultCatalogCollatorEntityProcessor(); + + describe('process', () => { + it('maps a returned entity', async () => { + const document = entityProcessor.process(entity, locationTemplate); + + expect(document).toMatchObject({ + title: entity.metadata.title, + location: '/catalog/namespace/component/test-entity', + text: entity.metadata.description, + namespace: entity.metadata.namespace, + componentType: entity.spec.type, + lifecycle: entity.spec.lifecycle, + owner: entity.spec.owner, + authorization: { + resourceRef: 'component:namespace/test-entity', + }, + }); + }); + + it('maps a returned entity with default fallback', async () => { + const entityWithoutTitle = { + ...entity, + metadata: { + ...entity.metadata, + title: undefined, + namespace: undefined, + }, + spec: { + type: undefined, + lifecycle: undefined, + owner: undefined, + }, + }; + + const document = entityProcessor.process( + entityWithoutTitle, + locationTemplate, + ); + + expect(document).toMatchObject({ + title: entity.metadata.name, + location: '/catalog/default/component/test-entity', + text: entity.metadata.description, + namespace: 'default', + componentType: 'other', + lifecycle: '', + owner: '', + authorization: { + resourceRef: 'component:default/test-entity', + }, + }); + }); + + it('maps a returned entity with custom locationTemplate', async () => { + const document = entityProcessor.process(entity, '/catalog/:name'); + + expect(document).toMatchObject({ + title: entity.metadata.title, + location: '/catalog/test-entity', + text: entity.metadata.description, + namespace: entity.metadata.namespace, + componentType: entity.spec.type, + lifecycle: entity.spec.lifecycle, + owner: entity.spec.owner, + authorization: { + resourceRef: 'component:namespace/test-entity', + }, + }); + }); + + it('maps a returned user entity', async () => { + const document = entityProcessor.process(userEntity, locationTemplate); + + expect(document).toMatchObject({ + title: userEntity.metadata.name, + location: '/catalog/default/user/test-user-entity', + text: `${userEntity.metadata.description} : ${userEntity.spec.profile.displayName}`, + namespace: 'default', + componentType: 'other', + lifecycle: '', + owner: '', + authorization: { + resourceRef: 'user:default/test-user-entity', + }, + }); + }); + + it('maps a returned user entity without display name', async () => { + const testEntity = { + ...userEntity, + spec: undefined, + }; + + const document = entityProcessor.process(testEntity, locationTemplate); + + expect(document).toMatchObject({ + title: userEntity.metadata.name, + location: '/catalog/default/user/test-user-entity', + text: userEntity.metadata.description, + namespace: 'default', + componentType: 'other', + lifecycle: '', + owner: '', + authorization: { + resourceRef: 'user:default/test-user-entity', + }, + }); + }); + + it('maps a returned group entity', async () => { + const groupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'test-group-entity', + description: 'The expected group description', + }, + spec: { + profile: { + displayName: 'Group 1', + }, + }, + }; + + const document = entityProcessor.process(groupEntity, locationTemplate); + + expect(document).toMatchObject({ + title: groupEntity.metadata.name, + location: '/catalog/default/group/test-group-entity', + text: `${groupEntity.metadata.description} : ${groupEntity.spec.profile.displayName}`, + namespace: 'default', + componentType: 'other', + lifecycle: '', + owner: '', + authorization: { + resourceRef: 'group:default/test-group-entity', + }, + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.ts new file mode 100644 index 0000000000..3078ed4310 --- /dev/null +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Entity, + isGroupEntity, + isUserEntity, + stringifyEntityRef, +} from '@backstage/catalog-model'; +import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; +import { CatalogCollatorEntityProcessor } from './CatalogCollatorEntityProcessor'; + +export class DefaultCatalogCollatorEntityProcessor + implements CatalogCollatorEntityProcessor +{ + public process( + entity: Entity, + locationTemplate: string, + ): CatalogEntityDocument { + return { + title: entity.metadata.title ?? entity.metadata.name, + location: this.applyArgsToFormat(locationTemplate, { + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + name: entity.metadata.name, + }), + text: this.getDocumentText(entity), + componentType: entity.spec?.type?.toString() || 'other', + type: entity.spec?.type?.toString() || 'other', + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: (entity.spec?.owner as string) || '', + authorization: { + resourceRef: stringifyEntityRef(entity), + }, + }; + } + + private applyArgsToFormat( + format: string, + args: Record, + ): string { + let formatted = format; + + for (const [key, value] of Object.entries(args)) { + formatted = formatted.replace(`:${key}`, value); + } + + return formatted.toLowerCase(); + } + + private getDocumentText(entity: Entity): string { + const documentTexts: string[] = []; + documentTexts.push(entity.metadata.description || ''); + + if (isUserEntity(entity) || isGroupEntity(entity)) { + if (entity.spec?.profile?.displayName) { + documentTexts.push(entity.spec.profile.displayName); + } + } + + return documentTexts.join(' : '); + } +} diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts index cf082c271c..8fbc2bef60 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts @@ -14,133 +14,27 @@ * limitations under the License. */ -import { - PluginEndpointDiscovery, - TokenManager, -} from '@backstage/backend-common'; -import { - CatalogApi, - CatalogClient, - GetEntitiesRequest, -} from '@backstage/catalog-client'; -import { stringifyEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; import { - catalogEntityReadPermission, - CatalogEntityDocument, -} from '@backstage/plugin-catalog-common'; -import { Permission } from '@backstage/plugin-permission-common'; -import { Readable } from 'stream'; -import { getDocumentText } from './util'; + CatalogCollatorFactory, + CatalogCollatorFactoryCreateOptions, +} from './CatalogCollatorFactory'; +import { DefaultCatalogCollatorEntityProcessor } from './DefaultCatalogCollatorEntityProcessor'; /** @public */ -export type DefaultCatalogCollatorFactoryOptions = { - discovery: PluginEndpointDiscovery; - tokenManager: TokenManager; - locationTemplate?: string; - filter?: GetEntitiesRequest['filter']; - batchSize?: number; - catalogClient?: CatalogApi; -}; +export type DefaultCatalogCollatorFactoryOptions = + CatalogCollatorFactoryCreateOptions; /** @public */ -export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { - public readonly type: string = 'software-catalog'; - public readonly visibilityPermission: Permission = - catalogEntityReadPermission; - - private locationTemplate: string; - private filter?: GetEntitiesRequest['filter']; - private batchSize: number; - private readonly catalogClient: CatalogApi; - private tokenManager: TokenManager; - +export class DefaultCatalogCollatorFactory extends CatalogCollatorFactory { static fromConfig( _config: Config, options: DefaultCatalogCollatorFactoryOptions, ) { - return new DefaultCatalogCollatorFactory(options); - } - - private constructor(options: DefaultCatalogCollatorFactoryOptions) { - const { - batchSize, - discovery, - locationTemplate, - filter, - catalogClient, - tokenManager, - } = options; - - this.locationTemplate = - locationTemplate || '/catalog/:namespace/:kind/:name'; - this.filter = filter; - this.batchSize = batchSize || 500; - this.catalogClient = - catalogClient || new CatalogClient({ discoveryApi: discovery }); - this.tokenManager = tokenManager; - } - - async getCollator(): Promise { - return Readable.from(this.execute()); - } - - private applyArgsToFormat( - format: string, - args: Record, - ): string { - let formatted = format; - for (const [key, value] of Object.entries(args)) { - formatted = formatted.replace(`:${key}`, value); - } - return formatted.toLowerCase(); - } - - private async *execute(): AsyncGenerator { - const { token } = await this.tokenManager.getToken(); - let entitiesRetrieved = 0; - let moreEntitiesToGet = true; - - // Offset/limit pagination is used on the Catalog Client in order to - // limit (and allow some control over) memory used by the search backend - // at index-time. - while (moreEntitiesToGet) { - const entities = ( - await this.catalogClient.getEntities( - { - filter: this.filter, - limit: this.batchSize, - offset: entitiesRetrieved, - }, - { token }, - ) - ).items; - - // Control looping through entity batches. - moreEntitiesToGet = entities.length === this.batchSize; - entitiesRetrieved += entities.length; - - for (const entity of entities) { - yield { - title: entity.metadata.title ?? entity.metadata.name, - location: this.applyArgsToFormat(this.locationTemplate, { - namespace: entity.metadata.namespace || 'default', - kind: entity.kind, - name: entity.metadata.name, - }), - text: getDocumentText(entity), - componentType: entity.spec?.type?.toString() || 'other', - type: entity.spec?.type?.toString() || 'other', - namespace: entity.metadata.namespace || 'default', - kind: entity.kind, - lifecycle: (entity.spec?.lifecycle as string) || '', - owner: (entity.spec?.owner as string) || '', - authorization: { - resourceRef: stringifyEntityRef(entity), - }, - }; - } - } + return new DefaultCatalogCollatorFactory({ + ...options, + type: 'software-catalog', + entityProcessor: new DefaultCatalogCollatorEntityProcessor(), + }); } } diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts index 2602f0fada..fd025ddca4 100644 --- a/plugins/catalog-backend/src/search/index.ts +++ b/plugins/catalog-backend/src/search/index.ts @@ -16,6 +16,11 @@ export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory'; +export { CatalogCollatorFactory } from './CatalogCollatorFactory'; +export type { + CatalogCollatorFactoryOptions, + CatalogCollatorFactoryCreateOptions, +} from './CatalogCollatorFactory'; /** * todo(backstage/techdocs-core): stop exporting this in a future release. diff --git a/plugins/catalog-backend/src/search/util.test.ts b/plugins/catalog-backend/src/search/util.test.ts deleted file mode 100644 index d2d1776338..0000000000 --- a/plugins/catalog-backend/src/search/util.test.ts +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - ComponentEntity, - GroupEntity, - UserEntity, -} from '@backstage/catalog-model'; -import { getDocumentText } from './util'; - -describe('getDocumentText', () => { - describe('kind is not User or Group', () => { - test('contains description if set', () => { - const entity = createComponent(); - entity.metadata.description = 'The expected description'; - const actual = getDocumentText(entity); - expect(actual).toContain(entity.metadata.description); - }); - - test('is empty if description is not set', () => { - const entity = createComponent(); - const actual = getDocumentText(entity); - expect(actual).toEqual(''); - }); - }); - - describe('kind is User', () => { - test('contains display name if set', () => { - const entity = createUser(); - const actual = getDocumentText(entity); - expect(actual).toContain(entity.spec.profile?.displayName); - }); - - test('contains description if set', () => { - const entity = createUser(); - const actual = getDocumentText(entity); - expect(actual).toContain(entity.metadata.description); - }); - - test('contains both description and display name if both are set', () => { - const entity = createUser(); - const actual = getDocumentText(entity); - expect(actual).toContain(entity.spec.profile?.displayName); - expect(actual).toContain(entity.metadata.description); - }); - - test('is empty if description and display name are not set', () => { - const entity = createUser(); - delete entity.metadata.description; - delete entity.spec.profile?.displayName; - const actual = getDocumentText(entity); - expect(actual).toEqual(''); - }); - }); - - describe('kind is Group', () => { - test('contains display name if set', () => { - const entity = createGroup(); - const actual = getDocumentText(entity); - expect(actual).toContain(entity.spec.profile?.displayName); - }); - - test('contains description if set', () => { - const entity = createGroup(); - const actual = getDocumentText(entity); - expect(actual).toContain(entity.metadata.description); - }); - - test('contains both description and display name if both are set', () => { - const entity = createGroup(); - const actual = getDocumentText(entity); - expect(actual).toContain(entity.spec.profile?.displayName); - expect(actual).toContain(entity.metadata.description); - }); - - test('is empty if description and display name are not set', () => { - const entity = createGroup(); - delete entity.metadata.description; - delete entity.spec.profile?.displayName; - const actual = getDocumentText(entity); - expect(actual).toEqual(''); - }); - }); -}); - -function createGroup(): GroupEntity { - return { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'group-1', - description: 'The expected description', - }, - spec: { - type: 'team', - profile: { - displayName: 'Group 1', - }, - children: [], - }, - }; -} - -function createUser(): UserEntity { - return { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'user-1', - description: 'The expected description', - }, - spec: { - profile: { - displayName: 'User 1', - }, - }, - }; -} - -function createComponent(): ComponentEntity { - return { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'component-1', - }, - spec: { - lifecycle: 'experimental', - owner: 'someone', - type: 'service', - }, - }; -} From 8ca614f9e34ce250e3f71ec2b65a636cda475ef1 Mon Sep 17 00:00:00 2001 From: Ilya Savich Date: Mon, 12 Dec 2022 14:08:02 +0100 Subject: [PATCH 2/3] draft Signed-off-by: Ilya Savich --- plugins/catalog-backend/api-report.md | 58 +++----- ...ts => CatalogCollatorEntityTransformer.ts} | 5 +- .../src/search/CatalogCollatorFactory.ts | 132 ------------------ ...tCatalogCollatorEntityTransformer.test.ts} | 29 ++-- ...efaultCatalogCollatorEntityTransformer.ts} | 8 +- .../DefaultCatalogCollatorFactory.test.ts | 46 +++++- .../search/DefaultCatalogCollatorFactory.ts | 110 +++++++++++++-- plugins/catalog-backend/src/search/index.ts | 6 +- 8 files changed, 187 insertions(+), 207 deletions(-) rename plugins/catalog-backend/src/search/{CatalogCollatorEntityProcessor.ts => CatalogCollatorEntityTransformer.ts} (83%) delete mode 100644 plugins/catalog-backend/src/search/CatalogCollatorFactory.ts rename plugins/catalog-backend/src/search/{DefaultCatalogCollatorEntityProcessor.test.ts => DefaultCatalogCollatorEntityTransformer.test.ts} (86%) rename plugins/catalog-backend/src/search/{DefaultCatalogCollatorEntityProcessor.ts => DefaultCatalogCollatorEntityTransformer.ts} (91%) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 8432092796..dfb3f94dc1 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -168,41 +168,11 @@ export class CatalogBuilder { } // @public (undocumented) -export abstract class CatalogCollatorFactory - implements DocumentCollatorFactory -{ - protected constructor(options: CatalogCollatorFactoryOptions); +export interface CatalogCollatorEntityTransformer { // (undocumented) - static fromConfig( - _config: Config, - _options: CatalogCollatorFactoryCreateOptions, - ): CatalogCollatorFactory; - // (undocumented) - getCollator(): Promise; - // (undocumented) - readonly type: string; - // (undocumented) - readonly visibilityPermission: Permission; + transform(entity: Entity, locationTemplate: string): CatalogEntityDocument; } -// @public (undocumented) -export type CatalogCollatorFactoryCreateOptions = Omit< - CatalogCollatorFactoryOptions, - 'entityProcessor' | 'type' ->; - -// @public (undocumented) -export type CatalogCollatorFactoryOptions = { - type: string; - discovery: PluginEndpointDiscovery; - tokenManager: TokenManager; - entityProcessor: CatalogCollatorEntityProcessor; - locationTemplate?: string; - filter?: GetEntitiesRequest['filter']; - batchSize?: number; - catalogClient?: CatalogApi; -}; - // @alpha export const catalogConditions: Conditions<{ hasAnnotation: PermissionRule< @@ -387,17 +357,31 @@ export class DefaultCatalogCollator { } // @public (undocumented) -export class DefaultCatalogCollatorFactory extends CatalogCollatorFactory { +export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { // (undocumented) static fromConfig( _config: Config, options: DefaultCatalogCollatorFactoryOptions, ): DefaultCatalogCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type: string; + // (undocumented) + readonly visibilityPermission: Permission; } // @public (undocumented) -export type DefaultCatalogCollatorFactoryOptions = - CatalogCollatorFactoryCreateOptions; +export type DefaultCatalogCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + type?: string; + locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; + batchSize?: number; + catalogClient?: CatalogApi; + entityTransformer?: CatalogCollatorEntityTransformer; +}; export { DeferredEntity }; @@ -607,8 +591,4 @@ export class UrlReaderProcessor implements CatalogProcessor { cache: CatalogProcessorCache, ): Promise; } - -// Warnings were encountered during analysis: -// -// src/search/CatalogCollatorFactory.d.ts:14:5 - (ae-forgotten-export) The symbol "CatalogCollatorEntityProcessor" needs to be exported by the entry point index.d.ts ``` diff --git a/plugins/catalog-backend/src/search/CatalogCollatorEntityProcessor.ts b/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts similarity index 83% rename from plugins/catalog-backend/src/search/CatalogCollatorEntityProcessor.ts rename to plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts index 0288cbec24..74615f40e9 100644 --- a/plugins/catalog-backend/src/search/CatalogCollatorEntityProcessor.ts +++ b/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts @@ -17,6 +17,7 @@ import { Entity } from '@backstage/catalog-model'; import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; -export interface CatalogCollatorEntityProcessor { - process(entity: Entity, locationTemplate: string): CatalogEntityDocument; +/** @public */ +export interface CatalogCollatorEntityTransformer { + transform(entity: Entity, locationTemplate: string): CatalogEntityDocument; } diff --git a/plugins/catalog-backend/src/search/CatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/CatalogCollatorFactory.ts deleted file mode 100644 index 8b9a60b264..0000000000 --- a/plugins/catalog-backend/src/search/CatalogCollatorFactory.ts +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - PluginEndpointDiscovery, - TokenManager, -} from '@backstage/backend-common'; -import { - CatalogApi, - CatalogClient, - GetEntitiesRequest, -} from '@backstage/catalog-client'; -import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; -import { - catalogEntityReadPermission, - CatalogEntityDocument, -} from '@backstage/plugin-catalog-common'; -import { Permission } from '@backstage/plugin-permission-common'; -import { Readable } from 'stream'; -import { CatalogCollatorEntityProcessor } from './CatalogCollatorEntityProcessor'; -import { Config } from '@backstage/config'; - -/** @public */ -export type CatalogCollatorFactoryOptions = { - type: string; - discovery: PluginEndpointDiscovery; - tokenManager: TokenManager; - entityProcessor: CatalogCollatorEntityProcessor; - locationTemplate?: string; - filter?: GetEntitiesRequest['filter']; - batchSize?: number; - catalogClient?: CatalogApi; -}; - -/** @public */ -export type CatalogCollatorFactoryCreateOptions = Omit< - CatalogCollatorFactoryOptions, - 'entityProcessor' | 'type' ->; - -/** @public */ -export abstract class CatalogCollatorFactory - implements DocumentCollatorFactory -{ - public readonly type: string; - public readonly visibilityPermission: Permission = - catalogEntityReadPermission; - - private locationTemplate: string; - private filter?: GetEntitiesRequest['filter']; - private batchSize: number; - private readonly catalogClient: CatalogApi; - private tokenManager: TokenManager; - private entityProcessor: CatalogCollatorEntityProcessor; - - static fromConfig( - _config: Config, - _options: CatalogCollatorFactoryCreateOptions, - ): CatalogCollatorFactory { - throw new Error('Method should be implemented'); - } - - protected constructor(options: CatalogCollatorFactoryOptions) { - const { - type, - batchSize, - discovery, - locationTemplate, - filter, - catalogClient, - tokenManager, - entityProcessor, - } = options; - - this.type = type; - this.locationTemplate = - locationTemplate || '/catalog/:namespace/:kind/:name'; - this.filter = filter; - this.batchSize = batchSize || 500; - this.catalogClient = - catalogClient || new CatalogClient({ discoveryApi: discovery }); - this.tokenManager = tokenManager; - this.entityProcessor = entityProcessor; - } - - async getCollator(): Promise { - return Readable.from(this.execute()); - } - - private async *execute(): AsyncGenerator { - const { token } = await this.tokenManager.getToken(); - let entitiesRetrieved = 0; - let moreEntitiesToGet = true; - - // Offset/limit pagination is used on the Catalog Client in order to - // limit (and allow some control over) memory used by the search backend - // at index-time. - while (moreEntitiesToGet) { - const entities = ( - await this.catalogClient.getEntities( - { - filter: this.filter, - limit: this.batchSize, - offset: entitiesRetrieved, - }, - { token }, - ) - ).items; - - // Control looping through entity batches. - moreEntitiesToGet = entities.length === this.batchSize; - entitiesRetrieved += entities.length; - - for (const entity of entities) { - yield this.entityProcessor.process(entity, this.locationTemplate); - } - } - } -} diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.test.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.test.ts similarity index 86% rename from plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.test.ts rename to plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.test.ts index 1183d5be97..648d287cdd 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.test.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { DefaultCatalogCollatorEntityProcessor } from './DefaultCatalogCollatorEntityProcessor'; +import { DefaultCatalogCollatorEntityTransformer } from './DefaultCatalogCollatorEntityTransformer'; const entity = { apiVersion: 'backstage.io/v1alpha1', @@ -48,12 +48,12 @@ const userEntity = { const locationTemplate = '/catalog/:namespace/:kind/:name'; -describe('DefaultCatalogCollatorEntityProcessor', () => { - const entityProcessor = new DefaultCatalogCollatorEntityProcessor(); +describe('DefaultCatalogCollatorEntityTransformer', () => { + const entityTransformer = new DefaultCatalogCollatorEntityTransformer(); - describe('process', () => { + describe('transform', () => { it('maps a returned entity', async () => { - const document = entityProcessor.process(entity, locationTemplate); + const document = entityTransformer.transform(entity, locationTemplate); expect(document).toMatchObject({ title: entity.metadata.title, @@ -84,7 +84,7 @@ describe('DefaultCatalogCollatorEntityProcessor', () => { }, }; - const document = entityProcessor.process( + const document = entityTransformer.transform( entityWithoutTitle, locationTemplate, ); @@ -104,7 +104,7 @@ describe('DefaultCatalogCollatorEntityProcessor', () => { }); it('maps a returned entity with custom locationTemplate', async () => { - const document = entityProcessor.process(entity, '/catalog/:name'); + const document = entityTransformer.transform(entity, '/catalog/:name'); expect(document).toMatchObject({ title: entity.metadata.title, @@ -121,7 +121,10 @@ describe('DefaultCatalogCollatorEntityProcessor', () => { }); it('maps a returned user entity', async () => { - const document = entityProcessor.process(userEntity, locationTemplate); + const document = entityTransformer.transform( + userEntity, + locationTemplate, + ); expect(document).toMatchObject({ title: userEntity.metadata.name, @@ -143,7 +146,10 @@ describe('DefaultCatalogCollatorEntityProcessor', () => { spec: undefined, }; - const document = entityProcessor.process(testEntity, locationTemplate); + const document = entityTransformer.transform( + testEntity, + locationTemplate, + ); expect(document).toMatchObject({ title: userEntity.metadata.name, @@ -174,7 +180,10 @@ describe('DefaultCatalogCollatorEntityProcessor', () => { }, }; - const document = entityProcessor.process(groupEntity, locationTemplate); + const document = entityTransformer.transform( + groupEntity, + locationTemplate, + ); expect(document).toMatchObject({ title: groupEntity.metadata.name, diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts similarity index 91% rename from plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.ts rename to plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts index 3078ed4310..08410c5eed 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts @@ -21,12 +21,12 @@ import { stringifyEntityRef, } from '@backstage/catalog-model'; import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; -import { CatalogCollatorEntityProcessor } from './CatalogCollatorEntityProcessor'; +import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; -export class DefaultCatalogCollatorEntityProcessor - implements CatalogCollatorEntityProcessor +export class DefaultCatalogCollatorEntityTransformer + implements CatalogCollatorEntityTransformer { - public process( + public transform( entity: Entity, locationTemplate: string, ): CatalogEntityDocument { diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts index 8b4e5601b4..0399764c34 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts @@ -95,9 +95,19 @@ describe('DefaultCatalogCollatorFactory', () => { ); }); - it('has expected type', () => { - const factory = DefaultCatalogCollatorFactory.fromConfig(config, options); - expect(factory.type).toBe('software-catalog'); + describe('type', () => { + it('has default', () => { + const factory = DefaultCatalogCollatorFactory.fromConfig(config, options); + expect(factory.type).toBe('software-catalog'); + }); + + it('has custom', () => { + const factory = DefaultCatalogCollatorFactory.fromConfig(config, { + ...options, + type: 'custom-type', + }); + expect(factory.type).toBe('custom-type'); + }); }); describe('getCollator', () => { @@ -150,6 +160,36 @@ describe('DefaultCatalogCollatorFactory', () => { }); }); + it('maps a returned entity to an expected CatalogEntityDocument with custom transformer', async () => { + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + + expect(documents[0]).toMatchObject({ + title: expectedEntities[0].metadata.name, + location: '/catalog/default/component/test-entity', + text: expectedEntities[0].metadata.description, + namespace: 'default', + componentType: expectedEntities[0]!.spec!.type, + lifecycle: expectedEntities[0]!.spec!.lifecycle, + owner: expectedEntities[0]!.spec!.owner, + authorization: { + resourceRef: 'component:default/test-entity', + }, + }); + expect(documents[1]).toMatchObject({ + title: expectedEntities[1].metadata.title, + location: '/catalog/default/component/test-entity-2', + text: expectedEntities[1].metadata.description, + namespace: 'default', + componentType: expectedEntities[1]!.spec!.type, + lifecycle: expectedEntities[1]!.spec!.lifecycle, + owner: expectedEntities[1]!.spec!.owner, + authorization: { + resourceRef: 'component:default/test-entity-2', + }, + }); + }); + it('maps a returned entity with a custom locationTemplate', async () => { // Provide an alternate location template. factory = DefaultCatalogCollatorFactory.fromConfig(new ConfigReader({}), { diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts index 8fbc2bef60..1b99624fdd 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts @@ -16,25 +16,111 @@ import { Config } from '@backstage/config'; import { - CatalogCollatorFactory, - CatalogCollatorFactoryCreateOptions, -} from './CatalogCollatorFactory'; -import { DefaultCatalogCollatorEntityProcessor } from './DefaultCatalogCollatorEntityProcessor'; + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { + CatalogApi, + CatalogClient, + GetEntitiesRequest, +} from '@backstage/catalog-client'; +import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; +import { + catalogEntityReadPermission, + CatalogEntityDocument, +} from '@backstage/plugin-catalog-common'; +import { Permission } from '@backstage/plugin-permission-common'; +import { Readable } from 'stream'; +import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; +import { DefaultCatalogCollatorEntityTransformer } from './DefaultCatalogCollatorEntityTransformer'; /** @public */ -export type DefaultCatalogCollatorFactoryOptions = - CatalogCollatorFactoryCreateOptions; +export type DefaultCatalogCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + type?: string; + locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; + batchSize?: number; + catalogClient?: CatalogApi; + entityTransformer?: CatalogCollatorEntityTransformer; +}; /** @public */ -export class DefaultCatalogCollatorFactory extends CatalogCollatorFactory { +export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { + public readonly type: string; + public readonly visibilityPermission: Permission = + catalogEntityReadPermission; + + private locationTemplate: string; + private filter?: GetEntitiesRequest['filter']; + private batchSize: number; + private readonly catalogClient: CatalogApi; + private tokenManager: TokenManager; + private entityTransformer: CatalogCollatorEntityTransformer; + static fromConfig( _config: Config, options: DefaultCatalogCollatorFactoryOptions, ) { - return new DefaultCatalogCollatorFactory({ - ...options, - type: 'software-catalog', - entityProcessor: new DefaultCatalogCollatorEntityProcessor(), - }); + return new DefaultCatalogCollatorFactory(options); + } + + private constructor(options: DefaultCatalogCollatorFactoryOptions) { + const { + batchSize, + discovery, + type, + locationTemplate, + filter, + catalogClient, + tokenManager, + entityTransformer, + } = options; + + this.type = type ?? 'software-catalog'; + this.locationTemplate = + locationTemplate || '/catalog/:namespace/:kind/:name'; + this.filter = filter; + this.batchSize = batchSize || 500; + this.catalogClient = + catalogClient || new CatalogClient({ discoveryApi: discovery }); + this.tokenManager = tokenManager; + this.entityTransformer = + entityTransformer ?? new DefaultCatalogCollatorEntityTransformer(); + } + + async getCollator(): Promise { + return Readable.from(this.execute()); + } + + private async *execute(): AsyncGenerator { + const { token } = await this.tokenManager.getToken(); + let entitiesRetrieved = 0; + let moreEntitiesToGet = true; + + // Offset/limit pagination is used on the Catalog Client in order to + // limit (and allow some control over) memory used by the search backend + // at index-time. + while (moreEntitiesToGet) { + const entities = ( + await this.catalogClient.getEntities( + { + filter: this.filter, + limit: this.batchSize, + offset: entitiesRetrieved, + }, + { token }, + ) + ).items; + + // Control looping through entity batches. + moreEntitiesToGet = entities.length === this.batchSize; + entitiesRetrieved += entities.length; + + for (const entity of entities) { + yield this.entityTransformer.transform(entity, this.locationTemplate); + } + } } } diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts index fd025ddca4..2675d7be1f 100644 --- a/plugins/catalog-backend/src/search/index.ts +++ b/plugins/catalog-backend/src/search/index.ts @@ -16,11 +16,7 @@ export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory'; -export { CatalogCollatorFactory } from './CatalogCollatorFactory'; -export type { - CatalogCollatorFactoryOptions, - CatalogCollatorFactoryCreateOptions, -} from './CatalogCollatorFactory'; +export type { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; /** * todo(backstage/techdocs-core): stop exporting this in a future release. From 860b1c0902b2f3c359706b3b380e5bb94341b5c7 Mon Sep 17 00:00:00 2001 From: Ilya Savich Date: Tue, 20 Dec 2022 13:16:13 +0100 Subject: [PATCH 3/3] Move authorization and location back, make transformer as a function Signed-off-by: Ilya Savich --- .changeset/clean-queens-judge.md | 2 +- docs/features/search/how-to-guides.md | 36 +++++++++ plugins/catalog-backend/api-report.md | 13 ++-- .../CatalogCollatorEntityTransformer.ts | 6 +- ...DefaultCatalogCollatorEntityTransformer.ts | 78 ------------------- .../DefaultCatalogCollatorFactory.test.ts | 78 +++++++++++-------- .../search/DefaultCatalogCollatorFactory.ts | 35 +++++++-- ...tCatalogCollatorEntityTransformer.test.ts} | 66 ++-------------- ...defaultCatalogCollatorEntityTransformer.ts | 46 +++++++++++ plugins/catalog-backend/src/search/index.ts | 1 + 10 files changed, 175 insertions(+), 186 deletions(-) delete mode 100644 plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts rename plugins/catalog-backend/src/search/{DefaultCatalogCollatorEntityTransformer.test.ts => defaultCatalogCollatorEntityTransformer.test.ts} (64%) create mode 100644 plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts diff --git a/.changeset/clean-queens-judge.md b/.changeset/clean-queens-judge.md index 22f87e6959..1dd7719c5f 100644 --- a/.changeset/clean-queens-judge.md +++ b/.changeset/clean-queens-judge.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend': patch --- -Refactored catalog collator, extracted abstract class and entity processor to make it possible to re-use functionality +The process of adding or modifying fields in the software-catalog search index has been simplified. For more details, see [how to customize fields in the Software Catalog index](../docs/features/search/how-to-guides.md#how-to-customize-fields-in-the-software-catalog-index). diff --git a/docs/features/search/how-to-guides.md b/docs/features/search/how-to-guides.md index 5cc2ab5f06..0eeb9b0454 100644 --- a/docs/features/search/how-to-guides.md +++ b/docs/features/search/how-to-guides.md @@ -114,6 +114,42 @@ of the `SearchType` component. > Check out the documentation around [integrating search into plugins](../../plugins/integrating-search-into-plugins.md#create-a-collator) for how to create your own collator. +## How to customize fields in the Software Catalog index + +Sometimes you will might want to have ability to control +which data passes to search index in catalog collator, or to customize data for specific kind. +You can easily do that by passing `entityTransformer` callback to `DefaultCatalogCollatorFactory`. +You can either just simply amend default behaviour, or even to write completely new document +(which should follow some required basic structure though). + +> `authorization` and `location` cannot be modified via a `entityTransformer`, `location` can be modified only through `locationTemplate`. + +```diff +// packages/backend/src/plugins/search.ts + +const entityTransformer: CatalogCollatorEntityTransformer = (entity: Entity) => { + if (entity.kind === 'SomeKind') { + return { + // customize here output for 'SomeKind' kind + }; + } + + return { + // and customize default output + ...defaultCatalogCollatorEntityTransformer(entity), + text: 'my super cool text', + }; +}; + +indexBuilder.addCollator({ + collator: DefaultCatalogCollatorFactory.fromConfig(env.config, { + discovery: env.discovery, + tokenManager: env.tokenManager, ++ entityTransformer, + }), +}); +``` + ## How to limit what can be searched in the Software Catalog The Software Catalog includes a wealth of information about the components, diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index dfb3f94dc1..1639c8c0cc 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -168,10 +168,9 @@ export class CatalogBuilder { } // @public (undocumented) -export interface CatalogCollatorEntityTransformer { - // (undocumented) - transform(entity: Entity, locationTemplate: string): CatalogEntityDocument; -} +export type CatalogCollatorEntityTransformer = ( + entity: Entity, +) => Omit; // @alpha export const catalogConditions: Conditions<{ @@ -356,6 +355,9 @@ export class DefaultCatalogCollator { readonly visibilityPermission: Permission; } +// @public (undocumented) +export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer; + // @public (undocumented) export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { // (undocumented) @@ -366,7 +368,7 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { // (undocumented) getCollator(): Promise; // (undocumented) - readonly type: string; + readonly type = 'software-catalog'; // (undocumented) readonly visibilityPermission: Permission; } @@ -375,7 +377,6 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { export type DefaultCatalogCollatorFactoryOptions = { discovery: PluginEndpointDiscovery; tokenManager: TokenManager; - type?: string; locationTemplate?: string; filter?: GetEntitiesRequest['filter']; batchSize?: number; diff --git a/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts b/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts index 74615f40e9..86d71d4c13 100644 --- a/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts +++ b/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts @@ -18,6 +18,6 @@ import { Entity } from '@backstage/catalog-model'; import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; /** @public */ -export interface CatalogCollatorEntityTransformer { - transform(entity: Entity, locationTemplate: string): CatalogEntityDocument; -} +export type CatalogCollatorEntityTransformer = ( + entity: Entity, +) => Omit; diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts deleted file mode 100644 index 08410c5eed..0000000000 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - Entity, - isGroupEntity, - isUserEntity, - stringifyEntityRef, -} from '@backstage/catalog-model'; -import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; -import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; - -export class DefaultCatalogCollatorEntityTransformer - implements CatalogCollatorEntityTransformer -{ - public transform( - entity: Entity, - locationTemplate: string, - ): CatalogEntityDocument { - return { - title: entity.metadata.title ?? entity.metadata.name, - location: this.applyArgsToFormat(locationTemplate, { - namespace: entity.metadata.namespace || 'default', - kind: entity.kind, - name: entity.metadata.name, - }), - text: this.getDocumentText(entity), - componentType: entity.spec?.type?.toString() || 'other', - type: entity.spec?.type?.toString() || 'other', - namespace: entity.metadata.namespace || 'default', - kind: entity.kind, - lifecycle: (entity.spec?.lifecycle as string) || '', - owner: (entity.spec?.owner as string) || '', - authorization: { - resourceRef: stringifyEntityRef(entity), - }, - }; - } - - private applyArgsToFormat( - format: string, - args: Record, - ): string { - let formatted = format; - - for (const [key, value] of Object.entries(args)) { - formatted = formatted.replace(`:${key}`, value); - } - - return formatted.toLowerCase(); - } - - private getDocumentText(entity: Entity): string { - const documentTexts: string[] = []; - documentTexts.push(entity.metadata.description || ''); - - if (isUserEntity(entity) || isGroupEntity(entity)) { - if (entity.spec?.profile?.displayName) { - documentTexts.push(entity.spec.profile.displayName); - } - } - - return documentTexts.join(' : '); - } -} diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts index 0399764c34..462e777dff 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts @@ -95,21 +95,6 @@ describe('DefaultCatalogCollatorFactory', () => { ); }); - describe('type', () => { - it('has default', () => { - const factory = DefaultCatalogCollatorFactory.fromConfig(config, options); - expect(factory.type).toBe('software-catalog'); - }); - - it('has custom', () => { - const factory = DefaultCatalogCollatorFactory.fromConfig(config, { - ...options, - type: 'custom-type', - }); - expect(factory.type).toBe('custom-type'); - }); - }); - describe('getCollator', () => { let factory: DefaultCatalogCollatorFactory; let collator: Readable; @@ -134,24 +119,28 @@ describe('DefaultCatalogCollatorFactory', () => { const pipeline = TestPipeline.fromCollator(collator); const { documents } = await pipeline.execute(); - expect(documents[0]).toMatchObject({ + expect(documents[0]).toEqual({ title: expectedEntities[0].metadata.name, location: '/catalog/default/component/test-entity', text: expectedEntities[0].metadata.description, namespace: 'default', componentType: expectedEntities[0]!.spec!.type, + kind: expectedEntities[0]!.kind, + type: expectedEntities[0]!.spec!.type, lifecycle: expectedEntities[0]!.spec!.lifecycle, owner: expectedEntities[0]!.spec!.owner, authorization: { resourceRef: 'component:default/test-entity', }, }); - expect(documents[1]).toMatchObject({ + expect(documents[1]).toEqual({ title: expectedEntities[1].metadata.title, location: '/catalog/default/component/test-entity-2', text: expectedEntities[1].metadata.description, namespace: 'default', componentType: expectedEntities[1]!.spec!.type, + kind: expectedEntities[1]!.kind, + type: expectedEntities[1]!.spec!.type, lifecycle: expectedEntities[1]!.spec!.lifecycle, owner: expectedEntities[1]!.spec!.owner, authorization: { @@ -161,29 +150,54 @@ describe('DefaultCatalogCollatorFactory', () => { }); it('maps a returned entity to an expected CatalogEntityDocument with custom transformer', async () => { - const pipeline = TestPipeline.fromCollator(collator); + const customFactory = DefaultCatalogCollatorFactory.fromConfig(config, { + ...options, + entityTransformer: entity => ({ + title: `custom-title-${ + entity.metadata.title ?? entity.metadata.name + }`, + namespace: 'custom/namespace', + text: 'custom-text', + type: 'custom-type', + componentType: 'custom-component-type', + kind: 'custom-kind', + lifecycle: 'custom-lifecycle', + owner: 'custom-owner', + authorization: { + resourceRef: 'custom:resource/ref', + }, + location: '/custom/location', + }), + }); + const customCollator = await customFactory.getCollator(); + + const pipeline = TestPipeline.fromCollator(customCollator); const { documents } = await pipeline.execute(); - expect(documents[0]).toMatchObject({ - title: expectedEntities[0].metadata.name, + expect(documents[0]).toEqual({ + title: 'custom-title-test-entity', location: '/catalog/default/component/test-entity', - text: expectedEntities[0].metadata.description, - namespace: 'default', - componentType: expectedEntities[0]!.spec!.type, - lifecycle: expectedEntities[0]!.spec!.lifecycle, - owner: expectedEntities[0]!.spec!.owner, + text: 'custom-text', + namespace: 'custom/namespace', + componentType: 'custom-component-type', + kind: 'custom-kind', + type: 'custom-type', + lifecycle: 'custom-lifecycle', + owner: 'custom-owner', authorization: { resourceRef: 'component:default/test-entity', }, }); - expect(documents[1]).toMatchObject({ - title: expectedEntities[1].metadata.title, + expect(documents[1]).toEqual({ + title: 'custom-title-Test Entity', location: '/catalog/default/component/test-entity-2', - text: expectedEntities[1].metadata.description, - namespace: 'default', - componentType: expectedEntities[1]!.spec!.type, - lifecycle: expectedEntities[1]!.spec!.lifecycle, - owner: expectedEntities[1]!.spec!.owner, + text: 'custom-text', + namespace: 'custom/namespace', + componentType: 'custom-component-type', + kind: 'custom-kind', + type: 'custom-type', + lifecycle: 'custom-lifecycle', + owner: 'custom-owner', authorization: { resourceRef: 'component:default/test-entity-2', }, diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts index 1b99624fdd..1c03836e51 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts @@ -32,13 +32,13 @@ import { import { Permission } from '@backstage/plugin-permission-common'; import { Readable } from 'stream'; import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; -import { DefaultCatalogCollatorEntityTransformer } from './DefaultCatalogCollatorEntityTransformer'; +import { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer'; +import { stringifyEntityRef } from '@backstage/catalog-model'; /** @public */ export type DefaultCatalogCollatorFactoryOptions = { discovery: PluginEndpointDiscovery; tokenManager: TokenManager; - type?: string; locationTemplate?: string; filter?: GetEntitiesRequest['filter']; batchSize?: number; @@ -48,7 +48,7 @@ export type DefaultCatalogCollatorFactoryOptions = { /** @public */ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { - public readonly type: string; + public readonly type = 'software-catalog'; public readonly visibilityPermission: Permission = catalogEntityReadPermission; @@ -70,7 +70,6 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { const { batchSize, discovery, - type, locationTemplate, filter, catalogClient, @@ -78,7 +77,6 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { entityTransformer, } = options; - this.type = type ?? 'software-catalog'; this.locationTemplate = locationTemplate || '/catalog/:namespace/:kind/:name'; this.filter = filter; @@ -87,7 +85,7 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { catalogClient || new CatalogClient({ discoveryApi: discovery }); this.tokenManager = tokenManager; this.entityTransformer = - entityTransformer ?? new DefaultCatalogCollatorEntityTransformer(); + entityTransformer ?? defaultCatalogCollatorEntityTransformer; } async getCollator(): Promise { @@ -119,8 +117,31 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { entitiesRetrieved += entities.length; for (const entity of entities) { - yield this.entityTransformer.transform(entity, this.locationTemplate); + yield { + ...this.entityTransformer(entity), + authorization: { + resourceRef: stringifyEntityRef(entity), + }, + location: this.applyArgsToFormat(this.locationTemplate, { + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + name: entity.metadata.name, + }), + }; } } } + + private applyArgsToFormat( + format: string, + args: Record, + ): string { + let formatted = format; + + for (const [key, value] of Object.entries(args)) { + formatted = formatted.replace(`:${key}`, value); + } + + return formatted.toLowerCase(); + } } diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.test.ts b/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.test.ts similarity index 64% rename from plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.test.ts rename to plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.test.ts index 648d287cdd..8206a58599 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.test.ts +++ b/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { DefaultCatalogCollatorEntityTransformer } from './DefaultCatalogCollatorEntityTransformer'; +import { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer'; const entity = { apiVersion: 'backstage.io/v1alpha1', @@ -46,26 +46,18 @@ const userEntity = { }, }; -const locationTemplate = '/catalog/:namespace/:kind/:name'; - describe('DefaultCatalogCollatorEntityTransformer', () => { - const entityTransformer = new DefaultCatalogCollatorEntityTransformer(); - describe('transform', () => { it('maps a returned entity', async () => { - const document = entityTransformer.transform(entity, locationTemplate); + const document = defaultCatalogCollatorEntityTransformer(entity); expect(document).toMatchObject({ title: entity.metadata.title, - location: '/catalog/namespace/component/test-entity', text: entity.metadata.description, namespace: entity.metadata.namespace, componentType: entity.spec.type, lifecycle: entity.spec.lifecycle, owner: entity.spec.owner, - authorization: { - resourceRef: 'component:namespace/test-entity', - }, }); }); @@ -84,59 +76,29 @@ describe('DefaultCatalogCollatorEntityTransformer', () => { }, }; - const document = entityTransformer.transform( - entityWithoutTitle, - locationTemplate, - ); + const document = + defaultCatalogCollatorEntityTransformer(entityWithoutTitle); expect(document).toMatchObject({ title: entity.metadata.name, - location: '/catalog/default/component/test-entity', text: entity.metadata.description, namespace: 'default', componentType: 'other', lifecycle: '', owner: '', - authorization: { - resourceRef: 'component:default/test-entity', - }, - }); - }); - - it('maps a returned entity with custom locationTemplate', async () => { - const document = entityTransformer.transform(entity, '/catalog/:name'); - - expect(document).toMatchObject({ - title: entity.metadata.title, - location: '/catalog/test-entity', - text: entity.metadata.description, - namespace: entity.metadata.namespace, - componentType: entity.spec.type, - lifecycle: entity.spec.lifecycle, - owner: entity.spec.owner, - authorization: { - resourceRef: 'component:namespace/test-entity', - }, }); }); it('maps a returned user entity', async () => { - const document = entityTransformer.transform( - userEntity, - locationTemplate, - ); + const document = defaultCatalogCollatorEntityTransformer(userEntity); expect(document).toMatchObject({ title: userEntity.metadata.name, - location: '/catalog/default/user/test-user-entity', text: `${userEntity.metadata.description} : ${userEntity.spec.profile.displayName}`, namespace: 'default', componentType: 'other', lifecycle: '', owner: '', - authorization: { - resourceRef: 'user:default/test-user-entity', - }, }); }); @@ -146,22 +108,15 @@ describe('DefaultCatalogCollatorEntityTransformer', () => { spec: undefined, }; - const document = entityTransformer.transform( - testEntity, - locationTemplate, - ); + const document = defaultCatalogCollatorEntityTransformer(testEntity); expect(document).toMatchObject({ title: userEntity.metadata.name, - location: '/catalog/default/user/test-user-entity', text: userEntity.metadata.description, namespace: 'default', componentType: 'other', lifecycle: '', owner: '', - authorization: { - resourceRef: 'user:default/test-user-entity', - }, }); }); @@ -180,22 +135,15 @@ describe('DefaultCatalogCollatorEntityTransformer', () => { }, }; - const document = entityTransformer.transform( - groupEntity, - locationTemplate, - ); + const document = defaultCatalogCollatorEntityTransformer(groupEntity); expect(document).toMatchObject({ title: groupEntity.metadata.name, - location: '/catalog/default/group/test-group-entity', text: `${groupEntity.metadata.description} : ${groupEntity.spec.profile.displayName}`, namespace: 'default', componentType: 'other', lifecycle: '', owner: '', - authorization: { - resourceRef: 'group:default/test-group-entity', - }, }); }); }); diff --git a/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts b/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts new file mode 100644 index 0000000000..2fc0c5d50d --- /dev/null +++ b/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, isGroupEntity, isUserEntity } from '@backstage/catalog-model'; +import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; + +const getDocumentText = (entity: Entity): string => { + const documentTexts: string[] = []; + documentTexts.push(entity.metadata.description || ''); + + if (isUserEntity(entity) || isGroupEntity(entity)) { + if (entity.spec?.profile?.displayName) { + documentTexts.push(entity.spec.profile.displayName); + } + } + + return documentTexts.join(' : '); +}; + +/** @public */ +export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer = + (entity: Entity) => { + return { + title: entity.metadata.title ?? entity.metadata.name, + text: getDocumentText(entity), + componentType: entity.spec?.type?.toString() || 'other', + type: entity.spec?.type?.toString() || 'other', + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: (entity.spec?.owner as string) || '', + }; + }; diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts index 2675d7be1f..c2a23e5132 100644 --- a/plugins/catalog-backend/src/search/index.ts +++ b/plugins/catalog-backend/src/search/index.ts @@ -17,6 +17,7 @@ export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory'; export type { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; +export type { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer'; /** * todo(backstage/techdocs-core): stop exporting this in a future release.