From 8ca614f9e34ce250e3f71ec2b65a636cda475ef1 Mon Sep 17 00:00:00 2001 From: Ilya Savich Date: Mon, 12 Dec 2022 14:08:02 +0100 Subject: [PATCH] draft Signed-off-by: Ilya Savich --- plugins/catalog-backend/api-report.md | 58 +++----- ...ts => CatalogCollatorEntityTransformer.ts} | 5 +- .../src/search/CatalogCollatorFactory.ts | 132 ------------------ ...tCatalogCollatorEntityTransformer.test.ts} | 29 ++-- ...efaultCatalogCollatorEntityTransformer.ts} | 8 +- .../DefaultCatalogCollatorFactory.test.ts | 46 +++++- .../search/DefaultCatalogCollatorFactory.ts | 110 +++++++++++++-- plugins/catalog-backend/src/search/index.ts | 6 +- 8 files changed, 187 insertions(+), 207 deletions(-) rename plugins/catalog-backend/src/search/{CatalogCollatorEntityProcessor.ts => CatalogCollatorEntityTransformer.ts} (83%) delete mode 100644 plugins/catalog-backend/src/search/CatalogCollatorFactory.ts rename plugins/catalog-backend/src/search/{DefaultCatalogCollatorEntityProcessor.test.ts => DefaultCatalogCollatorEntityTransformer.test.ts} (86%) rename plugins/catalog-backend/src/search/{DefaultCatalogCollatorEntityProcessor.ts => DefaultCatalogCollatorEntityTransformer.ts} (91%) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 8432092796..dfb3f94dc1 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -168,41 +168,11 @@ export class CatalogBuilder { } // @public (undocumented) -export abstract class CatalogCollatorFactory - implements DocumentCollatorFactory -{ - protected constructor(options: CatalogCollatorFactoryOptions); +export interface CatalogCollatorEntityTransformer { // (undocumented) - static fromConfig( - _config: Config, - _options: CatalogCollatorFactoryCreateOptions, - ): CatalogCollatorFactory; - // (undocumented) - getCollator(): Promise; - // (undocumented) - readonly type: string; - // (undocumented) - readonly visibilityPermission: Permission; + transform(entity: Entity, locationTemplate: string): CatalogEntityDocument; } -// @public (undocumented) -export type CatalogCollatorFactoryCreateOptions = Omit< - CatalogCollatorFactoryOptions, - 'entityProcessor' | 'type' ->; - -// @public (undocumented) -export type CatalogCollatorFactoryOptions = { - type: string; - discovery: PluginEndpointDiscovery; - tokenManager: TokenManager; - entityProcessor: CatalogCollatorEntityProcessor; - locationTemplate?: string; - filter?: GetEntitiesRequest['filter']; - batchSize?: number; - catalogClient?: CatalogApi; -}; - // @alpha export const catalogConditions: Conditions<{ hasAnnotation: PermissionRule< @@ -387,17 +357,31 @@ export class DefaultCatalogCollator { } // @public (undocumented) -export class DefaultCatalogCollatorFactory extends CatalogCollatorFactory { +export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { // (undocumented) static fromConfig( _config: Config, options: DefaultCatalogCollatorFactoryOptions, ): DefaultCatalogCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type: string; + // (undocumented) + readonly visibilityPermission: Permission; } // @public (undocumented) -export type DefaultCatalogCollatorFactoryOptions = - CatalogCollatorFactoryCreateOptions; +export type DefaultCatalogCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + type?: string; + locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; + batchSize?: number; + catalogClient?: CatalogApi; + entityTransformer?: CatalogCollatorEntityTransformer; +}; export { DeferredEntity }; @@ -607,8 +591,4 @@ export class UrlReaderProcessor implements CatalogProcessor { cache: CatalogProcessorCache, ): Promise; } - -// Warnings were encountered during analysis: -// -// src/search/CatalogCollatorFactory.d.ts:14:5 - (ae-forgotten-export) The symbol "CatalogCollatorEntityProcessor" needs to be exported by the entry point index.d.ts ``` diff --git a/plugins/catalog-backend/src/search/CatalogCollatorEntityProcessor.ts b/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts similarity index 83% rename from plugins/catalog-backend/src/search/CatalogCollatorEntityProcessor.ts rename to plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts index 0288cbec24..74615f40e9 100644 --- a/plugins/catalog-backend/src/search/CatalogCollatorEntityProcessor.ts +++ b/plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts @@ -17,6 +17,7 @@ import { Entity } from '@backstage/catalog-model'; import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; -export interface CatalogCollatorEntityProcessor { - process(entity: Entity, locationTemplate: string): CatalogEntityDocument; +/** @public */ +export interface CatalogCollatorEntityTransformer { + transform(entity: Entity, locationTemplate: string): CatalogEntityDocument; } diff --git a/plugins/catalog-backend/src/search/CatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/CatalogCollatorFactory.ts deleted file mode 100644 index 8b9a60b264..0000000000 --- a/plugins/catalog-backend/src/search/CatalogCollatorFactory.ts +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - PluginEndpointDiscovery, - TokenManager, -} from '@backstage/backend-common'; -import { - CatalogApi, - CatalogClient, - GetEntitiesRequest, -} from '@backstage/catalog-client'; -import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; -import { - catalogEntityReadPermission, - CatalogEntityDocument, -} from '@backstage/plugin-catalog-common'; -import { Permission } from '@backstage/plugin-permission-common'; -import { Readable } from 'stream'; -import { CatalogCollatorEntityProcessor } from './CatalogCollatorEntityProcessor'; -import { Config } from '@backstage/config'; - -/** @public */ -export type CatalogCollatorFactoryOptions = { - type: string; - discovery: PluginEndpointDiscovery; - tokenManager: TokenManager; - entityProcessor: CatalogCollatorEntityProcessor; - locationTemplate?: string; - filter?: GetEntitiesRequest['filter']; - batchSize?: number; - catalogClient?: CatalogApi; -}; - -/** @public */ -export type CatalogCollatorFactoryCreateOptions = Omit< - CatalogCollatorFactoryOptions, - 'entityProcessor' | 'type' ->; - -/** @public */ -export abstract class CatalogCollatorFactory - implements DocumentCollatorFactory -{ - public readonly type: string; - public readonly visibilityPermission: Permission = - catalogEntityReadPermission; - - private locationTemplate: string; - private filter?: GetEntitiesRequest['filter']; - private batchSize: number; - private readonly catalogClient: CatalogApi; - private tokenManager: TokenManager; - private entityProcessor: CatalogCollatorEntityProcessor; - - static fromConfig( - _config: Config, - _options: CatalogCollatorFactoryCreateOptions, - ): CatalogCollatorFactory { - throw new Error('Method should be implemented'); - } - - protected constructor(options: CatalogCollatorFactoryOptions) { - const { - type, - batchSize, - discovery, - locationTemplate, - filter, - catalogClient, - tokenManager, - entityProcessor, - } = options; - - this.type = type; - this.locationTemplate = - locationTemplate || '/catalog/:namespace/:kind/:name'; - this.filter = filter; - this.batchSize = batchSize || 500; - this.catalogClient = - catalogClient || new CatalogClient({ discoveryApi: discovery }); - this.tokenManager = tokenManager; - this.entityProcessor = entityProcessor; - } - - async getCollator(): Promise { - return Readable.from(this.execute()); - } - - private async *execute(): AsyncGenerator { - const { token } = await this.tokenManager.getToken(); - let entitiesRetrieved = 0; - let moreEntitiesToGet = true; - - // Offset/limit pagination is used on the Catalog Client in order to - // limit (and allow some control over) memory used by the search backend - // at index-time. - while (moreEntitiesToGet) { - const entities = ( - await this.catalogClient.getEntities( - { - filter: this.filter, - limit: this.batchSize, - offset: entitiesRetrieved, - }, - { token }, - ) - ).items; - - // Control looping through entity batches. - moreEntitiesToGet = entities.length === this.batchSize; - entitiesRetrieved += entities.length; - - for (const entity of entities) { - yield this.entityProcessor.process(entity, this.locationTemplate); - } - } - } -} diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.test.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.test.ts similarity index 86% rename from plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.test.ts rename to plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.test.ts index 1183d5be97..648d287cdd 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.test.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { DefaultCatalogCollatorEntityProcessor } from './DefaultCatalogCollatorEntityProcessor'; +import { DefaultCatalogCollatorEntityTransformer } from './DefaultCatalogCollatorEntityTransformer'; const entity = { apiVersion: 'backstage.io/v1alpha1', @@ -48,12 +48,12 @@ const userEntity = { const locationTemplate = '/catalog/:namespace/:kind/:name'; -describe('DefaultCatalogCollatorEntityProcessor', () => { - const entityProcessor = new DefaultCatalogCollatorEntityProcessor(); +describe('DefaultCatalogCollatorEntityTransformer', () => { + const entityTransformer = new DefaultCatalogCollatorEntityTransformer(); - describe('process', () => { + describe('transform', () => { it('maps a returned entity', async () => { - const document = entityProcessor.process(entity, locationTemplate); + const document = entityTransformer.transform(entity, locationTemplate); expect(document).toMatchObject({ title: entity.metadata.title, @@ -84,7 +84,7 @@ describe('DefaultCatalogCollatorEntityProcessor', () => { }, }; - const document = entityProcessor.process( + const document = entityTransformer.transform( entityWithoutTitle, locationTemplate, ); @@ -104,7 +104,7 @@ describe('DefaultCatalogCollatorEntityProcessor', () => { }); it('maps a returned entity with custom locationTemplate', async () => { - const document = entityProcessor.process(entity, '/catalog/:name'); + const document = entityTransformer.transform(entity, '/catalog/:name'); expect(document).toMatchObject({ title: entity.metadata.title, @@ -121,7 +121,10 @@ describe('DefaultCatalogCollatorEntityProcessor', () => { }); it('maps a returned user entity', async () => { - const document = entityProcessor.process(userEntity, locationTemplate); + const document = entityTransformer.transform( + userEntity, + locationTemplate, + ); expect(document).toMatchObject({ title: userEntity.metadata.name, @@ -143,7 +146,10 @@ describe('DefaultCatalogCollatorEntityProcessor', () => { spec: undefined, }; - const document = entityProcessor.process(testEntity, locationTemplate); + const document = entityTransformer.transform( + testEntity, + locationTemplate, + ); expect(document).toMatchObject({ title: userEntity.metadata.name, @@ -174,7 +180,10 @@ describe('DefaultCatalogCollatorEntityProcessor', () => { }, }; - const document = entityProcessor.process(groupEntity, locationTemplate); + const document = entityTransformer.transform( + groupEntity, + locationTemplate, + ); expect(document).toMatchObject({ title: groupEntity.metadata.name, diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts similarity index 91% rename from plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.ts rename to plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts index 3078ed4310..08410c5eed 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityProcessor.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts @@ -21,12 +21,12 @@ import { stringifyEntityRef, } from '@backstage/catalog-model'; import { CatalogEntityDocument } from '@backstage/plugin-catalog-common'; -import { CatalogCollatorEntityProcessor } from './CatalogCollatorEntityProcessor'; +import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; -export class DefaultCatalogCollatorEntityProcessor - implements CatalogCollatorEntityProcessor +export class DefaultCatalogCollatorEntityTransformer + implements CatalogCollatorEntityTransformer { - public process( + public transform( entity: Entity, locationTemplate: string, ): CatalogEntityDocument { diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts index 8b4e5601b4..0399764c34 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts @@ -95,9 +95,19 @@ describe('DefaultCatalogCollatorFactory', () => { ); }); - it('has expected type', () => { - const factory = DefaultCatalogCollatorFactory.fromConfig(config, options); - expect(factory.type).toBe('software-catalog'); + describe('type', () => { + it('has default', () => { + const factory = DefaultCatalogCollatorFactory.fromConfig(config, options); + expect(factory.type).toBe('software-catalog'); + }); + + it('has custom', () => { + const factory = DefaultCatalogCollatorFactory.fromConfig(config, { + ...options, + type: 'custom-type', + }); + expect(factory.type).toBe('custom-type'); + }); }); describe('getCollator', () => { @@ -150,6 +160,36 @@ describe('DefaultCatalogCollatorFactory', () => { }); }); + it('maps a returned entity to an expected CatalogEntityDocument with custom transformer', async () => { + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + + expect(documents[0]).toMatchObject({ + title: expectedEntities[0].metadata.name, + location: '/catalog/default/component/test-entity', + text: expectedEntities[0].metadata.description, + namespace: 'default', + componentType: expectedEntities[0]!.spec!.type, + lifecycle: expectedEntities[0]!.spec!.lifecycle, + owner: expectedEntities[0]!.spec!.owner, + authorization: { + resourceRef: 'component:default/test-entity', + }, + }); + expect(documents[1]).toMatchObject({ + title: expectedEntities[1].metadata.title, + location: '/catalog/default/component/test-entity-2', + text: expectedEntities[1].metadata.description, + namespace: 'default', + componentType: expectedEntities[1]!.spec!.type, + lifecycle: expectedEntities[1]!.spec!.lifecycle, + owner: expectedEntities[1]!.spec!.owner, + authorization: { + resourceRef: 'component:default/test-entity-2', + }, + }); + }); + it('maps a returned entity with a custom locationTemplate', async () => { // Provide an alternate location template. factory = DefaultCatalogCollatorFactory.fromConfig(new ConfigReader({}), { diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts index 8fbc2bef60..1b99624fdd 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts @@ -16,25 +16,111 @@ import { Config } from '@backstage/config'; import { - CatalogCollatorFactory, - CatalogCollatorFactoryCreateOptions, -} from './CatalogCollatorFactory'; -import { DefaultCatalogCollatorEntityProcessor } from './DefaultCatalogCollatorEntityProcessor'; + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { + CatalogApi, + CatalogClient, + GetEntitiesRequest, +} from '@backstage/catalog-client'; +import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; +import { + catalogEntityReadPermission, + CatalogEntityDocument, +} from '@backstage/plugin-catalog-common'; +import { Permission } from '@backstage/plugin-permission-common'; +import { Readable } from 'stream'; +import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; +import { DefaultCatalogCollatorEntityTransformer } from './DefaultCatalogCollatorEntityTransformer'; /** @public */ -export type DefaultCatalogCollatorFactoryOptions = - CatalogCollatorFactoryCreateOptions; +export type DefaultCatalogCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + type?: string; + locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; + batchSize?: number; + catalogClient?: CatalogApi; + entityTransformer?: CatalogCollatorEntityTransformer; +}; /** @public */ -export class DefaultCatalogCollatorFactory extends CatalogCollatorFactory { +export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { + public readonly type: string; + public readonly visibilityPermission: Permission = + catalogEntityReadPermission; + + private locationTemplate: string; + private filter?: GetEntitiesRequest['filter']; + private batchSize: number; + private readonly catalogClient: CatalogApi; + private tokenManager: TokenManager; + private entityTransformer: CatalogCollatorEntityTransformer; + static fromConfig( _config: Config, options: DefaultCatalogCollatorFactoryOptions, ) { - return new DefaultCatalogCollatorFactory({ - ...options, - type: 'software-catalog', - entityProcessor: new DefaultCatalogCollatorEntityProcessor(), - }); + return new DefaultCatalogCollatorFactory(options); + } + + private constructor(options: DefaultCatalogCollatorFactoryOptions) { + const { + batchSize, + discovery, + type, + locationTemplate, + filter, + catalogClient, + tokenManager, + entityTransformer, + } = options; + + this.type = type ?? 'software-catalog'; + this.locationTemplate = + locationTemplate || '/catalog/:namespace/:kind/:name'; + this.filter = filter; + this.batchSize = batchSize || 500; + this.catalogClient = + catalogClient || new CatalogClient({ discoveryApi: discovery }); + this.tokenManager = tokenManager; + this.entityTransformer = + entityTransformer ?? new DefaultCatalogCollatorEntityTransformer(); + } + + async getCollator(): Promise { + return Readable.from(this.execute()); + } + + private async *execute(): AsyncGenerator { + const { token } = await this.tokenManager.getToken(); + let entitiesRetrieved = 0; + let moreEntitiesToGet = true; + + // Offset/limit pagination is used on the Catalog Client in order to + // limit (and allow some control over) memory used by the search backend + // at index-time. + while (moreEntitiesToGet) { + const entities = ( + await this.catalogClient.getEntities( + { + filter: this.filter, + limit: this.batchSize, + offset: entitiesRetrieved, + }, + { token }, + ) + ).items; + + // Control looping through entity batches. + moreEntitiesToGet = entities.length === this.batchSize; + entitiesRetrieved += entities.length; + + for (const entity of entities) { + yield this.entityTransformer.transform(entity, this.locationTemplate); + } + } } } diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts index fd025ddca4..2675d7be1f 100644 --- a/plugins/catalog-backend/src/search/index.ts +++ b/plugins/catalog-backend/src/search/index.ts @@ -16,11 +16,7 @@ export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory'; -export { CatalogCollatorFactory } from './CatalogCollatorFactory'; -export type { - CatalogCollatorFactoryOptions, - CatalogCollatorFactoryCreateOptions, -} from './CatalogCollatorFactory'; +export type { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer'; /** * todo(backstage/techdocs-core): stop exporting this in a future release.