From 860b1c0902b2f3c359706b3b380e5bb94341b5c7 Mon Sep 17 00:00:00 2001 From: Ilya Savich Date: Tue, 20 Dec 2022 13:16:13 +0100 Subject: [PATCH] Move authorization and location back, make transformer as a function Signed-off-by: Ilya Savich --- .changeset/clean-queens-judge.md | 2 +- docs/features/search/how-to-guides.md | 36 +++++++++ plugins/catalog-backend/api-report.md | 13 ++-- .../CatalogCollatorEntityTransformer.ts | 6 +- ...DefaultCatalogCollatorEntityTransformer.ts | 78 ------------------- .../DefaultCatalogCollatorFactory.test.ts | 78 +++++++++++-------- .../search/DefaultCatalogCollatorFactory.ts | 35 +++++++-- ...tCatalogCollatorEntityTransformer.test.ts} | 66 ++-------------- ...defaultCatalogCollatorEntityTransformer.ts | 46 +++++++++++ plugins/catalog-backend/src/search/index.ts | 1 + 10 files changed, 175 insertions(+), 186 deletions(-) delete mode 100644 plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts rename plugins/catalog-backend/src/search/{DefaultCatalogCollatorEntityTransformer.test.ts => defaultCatalogCollatorEntityTransformer.test.ts} (64%) create mode 100644 plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts diff --git a/.changeset/clean-queens-judge.md b/.changeset/clean-queens-judge.md index 22f87e6959..1dd7719c5f 100644 --- a/.changeset/clean-queens-judge.md +++ b/.changeset/clean-queens-judge.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend': patch --- -Refactored catalog collator, extracted abstract class and entity processor to make it possible to re-use functionality +The process of adding or modifying fields in the software-catalog search index has been simplified. For more details, see [how to customize fields in the Software Catalog index](../docs/features/search/how-to-guides.md#how-to-customize-fields-in-the-software-catalog-index). diff --git a/docs/features/search/how-to-guides.md b/docs/features/search/how-to-guides.md index 5cc2ab5f06..0eeb9b0454 100644 --- a/docs/features/search/how-to-guides.md +++ b/docs/features/search/how-to-guides.md @@ -114,6 +114,42 @@ of the `SearchType` component. > Check out the documentation around [integrating search into plugins](../../plugins/integrating-search-into-plugins.md#create-a-collator) for how to create your own collator. +## How to customize fields in the Software Catalog index + +Sometimes you will might want to have ability to control +which data passes to search index in catalog collator, or to customize data for specific kind. +You can easily do that by passing `entityTransformer` callback to `DefaultCatalogCollatorFactory`. +You can either just simply amend default behaviour, or even to write completely new document +(which should follow some required basic structure though). + +> `authorization` and `location` cannot be modified via a `entityTransformer`, `location` can be modified only through `locationTemplate`. + +```diff +// packages/backend/src/plugins/search.ts + +const entityTransformer: CatalogCollatorEntityTransformer = (entity: Entity) => { + if (entity.kind === 'SomeKind') { + return { + // customize here output for 'SomeKind' kind + }; + } + + return { + // and customize default output + ...defaultCatalogCollatorEntityTransformer(entity), + text: 'my super cool text', + }; +}; + +indexBuilder.addCollator({ + collator: DefaultCatalogCollatorFactory.fromConfig(env.config, { + discovery: env.discovery, + tokenManager: env.tokenManager, ++ entityTransformer, + }), +}); +``` + ## How to limit what can be searched in the Software Catalog The Software Catalog includes a wealth of information about the components, diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index dfb3f94dc1..1639c8c0cc 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -168,10 +168,9 @@ export class CatalogBuilder { } // @public (undocumented) -export interface CatalogCollatorEntityTransformer { - // (undocumented) - transform(entity: Entity, locationTemplate: string): CatalogEntityDocument; -} +export type CatalogCollatorEntityTransformer = ( + entity: Entity, +) => Omit; // @alpha export const catalogConditions: Conditions<{ @@ -356,6 +355,9 @@ export class DefaultCatalogCollator { readonly visibilityPermission: Permission; } +// @public (undocumented) +export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer; + // @public (undocumented) export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { // (undocumented) @@ -366,7 +368,7 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { // (undocumented) getCollator(): Promise; // (undocumented) - readonly type: string; + readonly type = 'software-catalog'; // (undocumented) readonly visibilityPermission: Permission; } @@ -375,7 +377,6 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { export type DefaultCatalogCollatorFactoryOptions = { discovery: PluginEndpointDiscovery; tokenManager: TokenManager; - type?: string; locationTemplate?: string; filter?: GetEntitiesRequest['filter']; batchSize?: number; diff --git a/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts b/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts index 74615f40e9..86d71d4c13 100644 --- a/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts +++ b/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts @@ -18,6 +18,6 @@ import { Entity } from '@backstage/catalog-model'; import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; /** @public */ -export interface CatalogCollatorEntityTransformer { - transform(entity: Entity, locationTemplate: string): CatalogEntityDocument; -} +export type CatalogCollatorEntityTransformer = ( + entity: Entity, +) => Omit; diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts deleted file mode 100644 index 08410c5eed..0000000000 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - Entity, - isGroupEntity, - isUserEntity, - stringifyEntityRef, -} from '@backstage/catalog-model'; -import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; -import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; - -export class DefaultCatalogCollatorEntityTransformer - implements CatalogCollatorEntityTransformer -{ - public transform( - entity: Entity, - locationTemplate: string, - ): CatalogEntityDocument { - return { - title: entity.metadata.title ?? entity.metadata.name, - location: this.applyArgsToFormat(locationTemplate, { - namespace: entity.metadata.namespace || 'default', - kind: entity.kind, - name: entity.metadata.name, - }), - text: this.getDocumentText(entity), - componentType: entity.spec?.type?.toString() || 'other', - type: entity.spec?.type?.toString() || 'other', - namespace: entity.metadata.namespace || 'default', - kind: entity.kind, - lifecycle: (entity.spec?.lifecycle as string) || '', - owner: (entity.spec?.owner as string) || '', - authorization: { - resourceRef: stringifyEntityRef(entity), - }, - }; - } - - private applyArgsToFormat( - format: string, - args: Record, - ): string { - let formatted = format; - - for (const [key, value] of Object.entries(args)) { - formatted = formatted.replace(`:${key}`, value); - } - - return formatted.toLowerCase(); - } - - private getDocumentText(entity: Entity): string { - const documentTexts: string[] = []; - documentTexts.push(entity.metadata.description || ''); - - if (isUserEntity(entity) || isGroupEntity(entity)) { - if (entity.spec?.profile?.displayName) { - documentTexts.push(entity.spec.profile.displayName); - } - } - - return documentTexts.join(' : '); - } -} diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts index 0399764c34..462e777dff 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts @@ -95,21 +95,6 @@ describe('DefaultCatalogCollatorFactory', () => { ); }); - describe('type', () => { - it('has default', () => { - const factory = DefaultCatalogCollatorFactory.fromConfig(config, options); - expect(factory.type).toBe('software-catalog'); - }); - - it('has custom', () => { - const factory = DefaultCatalogCollatorFactory.fromConfig(config, { - ...options, - type: 'custom-type', - }); - expect(factory.type).toBe('custom-type'); - }); - }); - describe('getCollator', () => { let factory: DefaultCatalogCollatorFactory; let collator: Readable; @@ -134,24 +119,28 @@ describe('DefaultCatalogCollatorFactory', () => { const pipeline = TestPipeline.fromCollator(collator); const { documents } = await pipeline.execute(); - expect(documents[0]).toMatchObject({ + expect(documents[0]).toEqual({ title: expectedEntities[0].metadata.name, location: '/catalog/default/component/test-entity', text: expectedEntities[0].metadata.description, namespace: 'default', componentType: expectedEntities[0]!.spec!.type, + kind: expectedEntities[0]!.kind, + type: expectedEntities[0]!.spec!.type, lifecycle: expectedEntities[0]!.spec!.lifecycle, owner: expectedEntities[0]!.spec!.owner, authorization: { resourceRef: 'component:default/test-entity', }, }); - expect(documents[1]).toMatchObject({ + expect(documents[1]).toEqual({ title: expectedEntities[1].metadata.title, location: '/catalog/default/component/test-entity-2', text: expectedEntities[1].metadata.description, namespace: 'default', componentType: expectedEntities[1]!.spec!.type, + kind: expectedEntities[1]!.kind, + type: expectedEntities[1]!.spec!.type, lifecycle: expectedEntities[1]!.spec!.lifecycle, owner: expectedEntities[1]!.spec!.owner, authorization: { @@ -161,29 +150,54 @@ describe('DefaultCatalogCollatorFactory', () => { }); it('maps a returned entity to an expected CatalogEntityDocument with custom transformer', async () => { - const pipeline = TestPipeline.fromCollator(collator); + const customFactory = DefaultCatalogCollatorFactory.fromConfig(config, { + ...options, + entityTransformer: entity => ({ + title: `custom-title-${ + entity.metadata.title ?? entity.metadata.name + }`, + namespace: 'custom/namespace', + text: 'custom-text', + type: 'custom-type', + componentType: 'custom-component-type', + kind: 'custom-kind', + lifecycle: 'custom-lifecycle', + owner: 'custom-owner', + authorization: { + resourceRef: 'custom:resource/ref', + }, + location: '/custom/location', + }), + }); + const customCollator = await customFactory.getCollator(); + + const pipeline = TestPipeline.fromCollator(customCollator); const { documents } = await pipeline.execute(); - expect(documents[0]).toMatchObject({ - title: expectedEntities[0].metadata.name, + expect(documents[0]).toEqual({ + title: 'custom-title-test-entity', location: '/catalog/default/component/test-entity', - text: expectedEntities[0].metadata.description, - namespace: 'default', - componentType: expectedEntities[0]!.spec!.type, - lifecycle: expectedEntities[0]!.spec!.lifecycle, - owner: expectedEntities[0]!.spec!.owner, + text: 'custom-text', + namespace: 'custom/namespace', + componentType: 'custom-component-type', + kind: 'custom-kind', + type: 'custom-type', + lifecycle: 'custom-lifecycle', + owner: 'custom-owner', authorization: { resourceRef: 'component:default/test-entity', }, }); - expect(documents[1]).toMatchObject({ - title: expectedEntities[1].metadata.title, + expect(documents[1]).toEqual({ + title: 'custom-title-Test Entity', location: '/catalog/default/component/test-entity-2', - text: expectedEntities[1].metadata.description, - namespace: 'default', - componentType: expectedEntities[1]!.spec!.type, - lifecycle: expectedEntities[1]!.spec!.lifecycle, - owner: expectedEntities[1]!.spec!.owner, + text: 'custom-text', + namespace: 'custom/namespace', + componentType: 'custom-component-type', + kind: 'custom-kind', + type: 'custom-type', + lifecycle: 'custom-lifecycle', + owner: 'custom-owner', authorization: { resourceRef: 'component:default/test-entity-2', }, diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts index 1b99624fdd..1c03836e51 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts @@ -32,13 +32,13 @@ import { import { Permission } from '@backstage/plugin-permission-common'; import { Readable } from 'stream'; import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; -import { DefaultCatalogCollatorEntityTransformer } from './DefaultCatalogCollatorEntityTransformer'; +import { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer'; +import { stringifyEntityRef } from '@backstage/catalog-model'; /** @public */ export type DefaultCatalogCollatorFactoryOptions = { discovery: PluginEndpointDiscovery; tokenManager: TokenManager; - type?: string; locationTemplate?: string; filter?: GetEntitiesRequest['filter']; batchSize?: number; @@ -48,7 +48,7 @@ export type DefaultCatalogCollatorFactoryOptions = { /** @public */ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { - public readonly type: string; + public readonly type = 'software-catalog'; public readonly visibilityPermission: Permission = catalogEntityReadPermission; @@ -70,7 +70,6 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { const { batchSize, discovery, - type, locationTemplate, filter, catalogClient, @@ -78,7 +77,6 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { entityTransformer, } = options; - this.type = type ?? 'software-catalog'; this.locationTemplate = locationTemplate || '/catalog/:namespace/:kind/:name'; this.filter = filter; @@ -87,7 +85,7 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { catalogClient || new CatalogClient({ discoveryApi: discovery }); this.tokenManager = tokenManager; this.entityTransformer = - entityTransformer ?? new DefaultCatalogCollatorEntityTransformer(); + entityTransformer ?? defaultCatalogCollatorEntityTransformer; } async getCollator(): Promise { @@ -119,8 +117,31 @@ export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { entitiesRetrieved += entities.length; for (const entity of entities) { - yield this.entityTransformer.transform(entity, this.locationTemplate); + yield { + ...this.entityTransformer(entity), + authorization: { + resourceRef: stringifyEntityRef(entity), + }, + location: this.applyArgsToFormat(this.locationTemplate, { + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + name: entity.metadata.name, + }), + }; } } } + + private applyArgsToFormat( + format: string, + args: Record, + ): string { + let formatted = format; + + for (const [key, value] of Object.entries(args)) { + formatted = formatted.replace(`:${key}`, value); + } + + return formatted.toLowerCase(); + } } diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.test.ts b/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.test.ts similarity index 64% rename from plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.test.ts rename to plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.test.ts index 648d287cdd..8206a58599 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.test.ts +++ b/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { DefaultCatalogCollatorEntityTransformer } from './DefaultCatalogCollatorEntityTransformer'; +import { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer'; const entity = { apiVersion: 'backstage.io/v1alpha1', @@ -46,26 +46,18 @@ const userEntity = { }, }; -const locationTemplate = '/catalog/:namespace/:kind/:name'; - describe('DefaultCatalogCollatorEntityTransformer', () => { - const entityTransformer = new DefaultCatalogCollatorEntityTransformer(); - describe('transform', () => { it('maps a returned entity', async () => { - const document = entityTransformer.transform(entity, locationTemplate); + const document = defaultCatalogCollatorEntityTransformer(entity); expect(document).toMatchObject({ title: entity.metadata.title, - location: '/catalog/namespace/component/test-entity', text: entity.metadata.description, namespace: entity.metadata.namespace, componentType: entity.spec.type, lifecycle: entity.spec.lifecycle, owner: entity.spec.owner, - authorization: { - resourceRef: 'component:namespace/test-entity', - }, }); }); @@ -84,59 +76,29 @@ describe('DefaultCatalogCollatorEntityTransformer', () => { }, }; - const document = entityTransformer.transform( - entityWithoutTitle, - locationTemplate, - ); + const document = + defaultCatalogCollatorEntityTransformer(entityWithoutTitle); expect(document).toMatchObject({ title: entity.metadata.name, - location: '/catalog/default/component/test-entity', text: entity.metadata.description, namespace: 'default', componentType: 'other', lifecycle: '', owner: '', - authorization: { - resourceRef: 'component:default/test-entity', - }, - }); - }); - - it('maps a returned entity with custom locationTemplate', async () => { - const document = entityTransformer.transform(entity, '/catalog/:name'); - - expect(document).toMatchObject({ - title: entity.metadata.title, - location: '/catalog/test-entity', - text: entity.metadata.description, - namespace: entity.metadata.namespace, - componentType: entity.spec.type, - lifecycle: entity.spec.lifecycle, - owner: entity.spec.owner, - authorization: { - resourceRef: 'component:namespace/test-entity', - }, }); }); it('maps a returned user entity', async () => { - const document = entityTransformer.transform( - userEntity, - locationTemplate, - ); + const document = defaultCatalogCollatorEntityTransformer(userEntity); expect(document).toMatchObject({ title: userEntity.metadata.name, - location: '/catalog/default/user/test-user-entity', text: `${userEntity.metadata.description} : ${userEntity.spec.profile.displayName}`, namespace: 'default', componentType: 'other', lifecycle: '', owner: '', - authorization: { - resourceRef: 'user:default/test-user-entity', - }, }); }); @@ -146,22 +108,15 @@ describe('DefaultCatalogCollatorEntityTransformer', () => { spec: undefined, }; - const document = entityTransformer.transform( - testEntity, - locationTemplate, - ); + const document = defaultCatalogCollatorEntityTransformer(testEntity); expect(document).toMatchObject({ title: userEntity.metadata.name, - location: '/catalog/default/user/test-user-entity', text: userEntity.metadata.description, namespace: 'default', componentType: 'other', lifecycle: '', owner: '', - authorization: { - resourceRef: 'user:default/test-user-entity', - }, }); }); @@ -180,22 +135,15 @@ describe('DefaultCatalogCollatorEntityTransformer', () => { }, }; - const document = entityTransformer.transform( - groupEntity, - locationTemplate, - ); + const document = defaultCatalogCollatorEntityTransformer(groupEntity); expect(document).toMatchObject({ title: groupEntity.metadata.name, - location: '/catalog/default/group/test-group-entity', text: `${groupEntity.metadata.description} : ${groupEntity.spec.profile.displayName}`, namespace: 'default', componentType: 'other', lifecycle: '', owner: '', - authorization: { - resourceRef: 'group:default/test-group-entity', - }, }); }); }); diff --git a/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts b/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts new file mode 100644 index 0000000000..2fc0c5d50d --- /dev/null +++ b/plugins/catalog-backend/src/search/defaultCatalogCollatorEntityTransformer.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, isGroupEntity, isUserEntity } from '@backstage/catalog-model'; +import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; + +const getDocumentText = (entity: Entity): string => { + const documentTexts: string[] = []; + documentTexts.push(entity.metadata.description || ''); + + if (isUserEntity(entity) || isGroupEntity(entity)) { + if (entity.spec?.profile?.displayName) { + documentTexts.push(entity.spec.profile.displayName); + } + } + + return documentTexts.join(' : '); +}; + +/** @public */ +export const defaultCatalogCollatorEntityTransformer: CatalogCollatorEntityTransformer = + (entity: Entity) => { + return { + title: entity.metadata.title ?? entity.metadata.name, + text: getDocumentText(entity), + componentType: entity.spec?.type?.toString() || 'other', + type: entity.spec?.type?.toString() || 'other', + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: (entity.spec?.owner as string) || '', + }; + }; diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts index 2675d7be1f..c2a23e5132 100644 --- a/plugins/catalog-backend/src/search/index.ts +++ b/plugins/catalog-backend/src/search/index.ts @@ -17,6 +17,7 @@ export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory'; export type { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; +export type { defaultCatalogCollatorEntityTransformer } from './defaultCatalogCollatorEntityTransformer'; /** * todo(backstage/techdocs-core): stop exporting this in a future release.