draft

Signed-off-by: Ilya Savich <isavich@box.com>

plugins/catalog-backend/api-report.md

@@ -168,41 +168,11 @@ export class CatalogBuilder {
 }
 
 // @public (undocumented)
-export abstract class CatalogCollatorFactory
-  implements DocumentCollatorFactory
-{
-  protected constructor(options: CatalogCollatorFactoryOptions);
+export interface CatalogCollatorEntityTransformer {
   // (undocumented)
-  static fromConfig(
-    _config: Config,
-    _options: CatalogCollatorFactoryCreateOptions,
-  ): CatalogCollatorFactory;
-  // (undocumented)
-  getCollator(): Promise<Readable>;
-  // (undocumented)
-  readonly type: string;
-  // (undocumented)
-  readonly visibilityPermission: Permission;
+  transform(entity: Entity, locationTemplate: string): CatalogEntityDocument;
 }
 
-// @public (undocumented)
-export type CatalogCollatorFactoryCreateOptions = Omit<
-  CatalogCollatorFactoryOptions,
-  'entityProcessor' | 'type'
->;
-
-// @public (undocumented)
-export type CatalogCollatorFactoryOptions = {
-  type: string;
-  discovery: PluginEndpointDiscovery;
-  tokenManager: TokenManager;
-  entityProcessor: CatalogCollatorEntityProcessor;
-  locationTemplate?: string;
-  filter?: GetEntitiesRequest['filter'];
-  batchSize?: number;
-  catalogClient?: CatalogApi;
-};
-
 // @alpha
 export const catalogConditions: Conditions<{
   hasAnnotation: PermissionRule<
@@ -387,17 +357,31 @@ export class DefaultCatalogCollator {
 }
 
 // @public (undocumented)
-export class DefaultCatalogCollatorFactory extends CatalogCollatorFactory {
+export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
   // (undocumented)
   static fromConfig(
     _config: Config,
     options: DefaultCatalogCollatorFactoryOptions,
   ): DefaultCatalogCollatorFactory;
+  // (undocumented)
+  getCollator(): Promise<Readable>;
+  // (undocumented)
+  readonly type: string;
+  // (undocumented)
+  readonly visibilityPermission: Permission;
 }
 
 // @public (undocumented)
-export type DefaultCatalogCollatorFactoryOptions =
-  CatalogCollatorFactoryCreateOptions;
+export type DefaultCatalogCollatorFactoryOptions = {
+  discovery: PluginEndpointDiscovery;
+  tokenManager: TokenManager;
+  type?: string;
+  locationTemplate?: string;
+  filter?: GetEntitiesRequest['filter'];
+  batchSize?: number;
+  catalogClient?: CatalogApi;
+  entityTransformer?: CatalogCollatorEntityTransformer;
+};
 
 export { DeferredEntity };
 
@@ -607,8 +591,4 @@ export class UrlReaderProcessor implements CatalogProcessor {
     cache: CatalogProcessorCache,
   ): Promise<boolean>;
 }
-
-// Warnings were encountered during analysis:
-//
-// src/search/CatalogCollatorFactory.d.ts:14:5 - (ae-forgotten-export) The symbol "CatalogCollatorEntityProcessor" needs to be exported by the entry point index.d.ts
 ```

plugins/catalog-backend/src/search/CatalogCollatorEntityTransformer.ts

@@ -17,6 +17,7 @@
 import { Entity } from '@backstage/catalog-model';
 import { CatalogEntityDocument } from '@backstage/plugin-catalog-common';
 
-export interface CatalogCollatorEntityProcessor {
-  process(entity: Entity, locationTemplate: string): CatalogEntityDocument;
+/** @public */
+export interface CatalogCollatorEntityTransformer {
+  transform(entity: Entity, locationTemplate: string): CatalogEntityDocument;
 }

plugins/catalog-backend/src/search/CatalogCollatorFactory.ts

@@ -1,132 +0,0 @@
-/*
- * Copyright 2022 The Backstage Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import {
-  PluginEndpointDiscovery,
-  TokenManager,
-} from '@backstage/backend-common';
-import {
-  CatalogApi,
-  CatalogClient,
-  GetEntitiesRequest,
-} from '@backstage/catalog-client';
-import { DocumentCollatorFactory } from '@backstage/plugin-search-common';
-import {
-  catalogEntityReadPermission,
-  CatalogEntityDocument,
-} from '@backstage/plugin-catalog-common';
-import { Permission } from '@backstage/plugin-permission-common';
-import { Readable } from 'stream';
-import { CatalogCollatorEntityProcessor } from './CatalogCollatorEntityProcessor';
-import { Config } from '@backstage/config';
-
-/** @public */
-export type CatalogCollatorFactoryOptions = {
-  type: string;
-  discovery: PluginEndpointDiscovery;
-  tokenManager: TokenManager;
-  entityProcessor: CatalogCollatorEntityProcessor;
-  locationTemplate?: string;
-  filter?: GetEntitiesRequest['filter'];
-  batchSize?: number;
-  catalogClient?: CatalogApi;
-};
-
-/** @public */
-export type CatalogCollatorFactoryCreateOptions = Omit<
-  CatalogCollatorFactoryOptions,
-  'entityProcessor' | 'type'
->;
-
-/** @public */
-export abstract class CatalogCollatorFactory
-  implements DocumentCollatorFactory
-{
-  public readonly type: string;
-  public readonly visibilityPermission: Permission =
-    catalogEntityReadPermission;
-
-  private locationTemplate: string;
-  private filter?: GetEntitiesRequest['filter'];
-  private batchSize: number;
-  private readonly catalogClient: CatalogApi;
-  private tokenManager: TokenManager;
-  private entityProcessor: CatalogCollatorEntityProcessor;
-
-  static fromConfig(
-    _config: Config,
-    _options: CatalogCollatorFactoryCreateOptions,
-  ): CatalogCollatorFactory {
-    throw new Error('Method should be implemented');
-  }
-
-  protected constructor(options: CatalogCollatorFactoryOptions) {
-    const {
-      type,
-      batchSize,
-      discovery,
-      locationTemplate,
-      filter,
-      catalogClient,
-      tokenManager,
-      entityProcessor,
-    } = options;
-
-    this.type = type;
-    this.locationTemplate =
-      locationTemplate || '/catalog/:namespace/:kind/:name';
-    this.filter = filter;
-    this.batchSize = batchSize || 500;
-    this.catalogClient =
-      catalogClient || new CatalogClient({ discoveryApi: discovery });
-    this.tokenManager = tokenManager;
-    this.entityProcessor = entityProcessor;
-  }
-
-  async getCollator(): Promise<Readable> {
-    return Readable.from(this.execute());
-  }
-
-  private async *execute(): AsyncGenerator<CatalogEntityDocument> {
-    const { token } = await this.tokenManager.getToken();
-    let entitiesRetrieved = 0;
-    let moreEntitiesToGet = true;
-
-    // Offset/limit pagination is used on the Catalog Client in order to
-    // limit (and allow some control over) memory used by the search backend
-    // at index-time.
-    while (moreEntitiesToGet) {
-      const entities = (
-        await this.catalogClient.getEntities(
-          {
-            filter: this.filter,
-            limit: this.batchSize,
-            offset: entitiesRetrieved,
-          },
-          { token },
-        )
-      ).items;
-
-      // Control looping through entity batches.
-      moreEntitiesToGet = entities.length === this.batchSize;
-      entitiesRetrieved += entities.length;
-
-      for (const entity of entities) {
-        yield this.entityProcessor.process(entity, this.locationTemplate);
-      }
-    }
-  }
-}

plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.test.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { DefaultCatalogCollatorEntityProcessor } from './DefaultCatalogCollatorEntityProcessor';
+import { DefaultCatalogCollatorEntityTransformer } from './DefaultCatalogCollatorEntityTransformer';
 
 const entity = {
   apiVersion: 'backstage.io/v1alpha1',
@@ -48,12 +48,12 @@ const userEntity = {
 
 const locationTemplate = '/catalog/:namespace/:kind/:name';
 
-describe('DefaultCatalogCollatorEntityProcessor', () => {
-  const entityProcessor = new DefaultCatalogCollatorEntityProcessor();
+describe('DefaultCatalogCollatorEntityTransformer', () => {
+  const entityTransformer = new DefaultCatalogCollatorEntityTransformer();
 
-  describe('process', () => {
+  describe('transform', () => {
     it('maps a returned entity', async () => {
-      const document = entityProcessor.process(entity, locationTemplate);
+      const document = entityTransformer.transform(entity, locationTemplate);
 
       expect(document).toMatchObject({
         title: entity.metadata.title,
@@ -84,7 +84,7 @@ describe('DefaultCatalogCollatorEntityProcessor', () => {
         },
       };
 
-      const document = entityProcessor.process(
+      const document = entityTransformer.transform(
         entityWithoutTitle,
         locationTemplate,
       );
@@ -104,7 +104,7 @@ describe('DefaultCatalogCollatorEntityProcessor', () => {
     });
 
     it('maps a returned entity with custom locationTemplate', async () => {
-      const document = entityProcessor.process(entity, '/catalog/:name');
+      const document = entityTransformer.transform(entity, '/catalog/:name');
 
       expect(document).toMatchObject({
         title: entity.metadata.title,
@@ -121,7 +121,10 @@ describe('DefaultCatalogCollatorEntityProcessor', () => {
     });
 
     it('maps a returned user entity', async () => {
-      const document = entityProcessor.process(userEntity, locationTemplate);
+      const document = entityTransformer.transform(
+        userEntity,
+        locationTemplate,
+      );
 
       expect(document).toMatchObject({
         title: userEntity.metadata.name,
@@ -143,7 +146,10 @@ describe('DefaultCatalogCollatorEntityProcessor', () => {
         spec: undefined,
       };
 
-      const document = entityProcessor.process(testEntity, locationTemplate);
+      const document = entityTransformer.transform(
+        testEntity,
+        locationTemplate,
+      );
 
       expect(document).toMatchObject({
         title: userEntity.metadata.name,
@@ -174,7 +180,10 @@ describe('DefaultCatalogCollatorEntityProcessor', () => {
         },
       };
 
-      const document = entityProcessor.process(groupEntity, locationTemplate);
+      const document = entityTransformer.transform(
+        groupEntity,
+        locationTemplate,
+      );
 
       expect(document).toMatchObject({
         title: groupEntity.metadata.name,

plugins/catalog-backend/src/search/DefaultCatalogCollatorEntityTransformer.ts

@@ -21,12 +21,12 @@ import {
   stringifyEntityRef,
 } from '@backstage/catalog-model';
 import { CatalogEntityDocument } from '@backstage/plugin-catalog-common';
-import { CatalogCollatorEntityProcessor } from './CatalogCollatorEntityProcessor';
+import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer';
 
-export class DefaultCatalogCollatorEntityProcessor
-  implements CatalogCollatorEntityProcessor
+export class DefaultCatalogCollatorEntityTransformer
+  implements CatalogCollatorEntityTransformer
 {
-  public process(
+  public transform(
     entity: Entity,
     locationTemplate: string,
   ): CatalogEntityDocument {

plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts

@@ -95,9 +95,19 @@ describe('DefaultCatalogCollatorFactory', () => {
     );
   });
 
-  it('has expected type', () => {
-    const factory = DefaultCatalogCollatorFactory.fromConfig(config, options);
-    expect(factory.type).toBe('software-catalog');
+  describe('type', () => {
+    it('has default', () => {
+      const factory = DefaultCatalogCollatorFactory.fromConfig(config, options);
+      expect(factory.type).toBe('software-catalog');
+    });
+
+    it('has custom', () => {
+      const factory = DefaultCatalogCollatorFactory.fromConfig(config, {
+        ...options,
+        type: 'custom-type',
+      });
+      expect(factory.type).toBe('custom-type');
+    });
   });
 
   describe('getCollator', () => {
@@ -150,6 +160,36 @@ describe('DefaultCatalogCollatorFactory', () => {
       });
     });
 
+    it('maps a returned entity to an expected CatalogEntityDocument with custom transformer', async () => {
+      const pipeline = TestPipeline.fromCollator(collator);
+      const { documents } = await pipeline.execute();
+
+      expect(documents[0]).toMatchObject({
+        title: expectedEntities[0].metadata.name,
+        location: '/catalog/default/component/test-entity',
+        text: expectedEntities[0].metadata.description,
+        namespace: 'default',
+        componentType: expectedEntities[0]!.spec!.type,
+        lifecycle: expectedEntities[0]!.spec!.lifecycle,
+        owner: expectedEntities[0]!.spec!.owner,
+        authorization: {
+          resourceRef: 'component:default/test-entity',
+        },
+      });
+      expect(documents[1]).toMatchObject({
+        title: expectedEntities[1].metadata.title,
+        location: '/catalog/default/component/test-entity-2',
+        text: expectedEntities[1].metadata.description,
+        namespace: 'default',
+        componentType: expectedEntities[1]!.spec!.type,
+        lifecycle: expectedEntities[1]!.spec!.lifecycle,
+        owner: expectedEntities[1]!.spec!.owner,
+        authorization: {
+          resourceRef: 'component:default/test-entity-2',
+        },
+      });
+    });
+
     it('maps a returned entity with a custom locationTemplate', async () => {
       // Provide an alternate location template.
       factory = DefaultCatalogCollatorFactory.fromConfig(new ConfigReader({}), {

plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts

@@ -16,25 +16,111 @@
 
 import { Config } from '@backstage/config';
 import {
-  CatalogCollatorFactory,
-  CatalogCollatorFactoryCreateOptions,
-} from './CatalogCollatorFactory';
-import { DefaultCatalogCollatorEntityProcessor } from './DefaultCatalogCollatorEntityProcessor';
+  PluginEndpointDiscovery,
+  TokenManager,
+} from '@backstage/backend-common';
+import {
+  CatalogApi,
+  CatalogClient,
+  GetEntitiesRequest,
+} from '@backstage/catalog-client';
+import { DocumentCollatorFactory } from '@backstage/plugin-search-common';
+import {
+  catalogEntityReadPermission,
+  CatalogEntityDocument,
+} from '@backstage/plugin-catalog-common';
+import { Permission } from '@backstage/plugin-permission-common';
+import { Readable } from 'stream';
+import { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer';
+import { DefaultCatalogCollatorEntityTransformer } from './DefaultCatalogCollatorEntityTransformer';
 
 /** @public */
-export type DefaultCatalogCollatorFactoryOptions =
-  CatalogCollatorFactoryCreateOptions;
+export type DefaultCatalogCollatorFactoryOptions = {
+  discovery: PluginEndpointDiscovery;
+  tokenManager: TokenManager;
+  type?: string;
+  locationTemplate?: string;
+  filter?: GetEntitiesRequest['filter'];
+  batchSize?: number;
+  catalogClient?: CatalogApi;
+  entityTransformer?: CatalogCollatorEntityTransformer;
+};
 
 /** @public */
-export class DefaultCatalogCollatorFactory extends CatalogCollatorFactory {
+export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory {
+  public readonly type: string;
+  public readonly visibilityPermission: Permission =
+    catalogEntityReadPermission;
+
+  private locationTemplate: string;
+  private filter?: GetEntitiesRequest['filter'];
+  private batchSize: number;
+  private readonly catalogClient: CatalogApi;
+  private tokenManager: TokenManager;
+  private entityTransformer: CatalogCollatorEntityTransformer;
+
   static fromConfig(
     _config: Config,
     options: DefaultCatalogCollatorFactoryOptions,
   ) {
-    return new DefaultCatalogCollatorFactory({
-      ...options,
-      type: 'software-catalog',
-      entityProcessor: new DefaultCatalogCollatorEntityProcessor(),
-    });
+    return new DefaultCatalogCollatorFactory(options);
+  }
+
+  private constructor(options: DefaultCatalogCollatorFactoryOptions) {
+    const {
+      batchSize,
+      discovery,
+      type,
+      locationTemplate,
+      filter,
+      catalogClient,
+      tokenManager,
+      entityTransformer,
+    } = options;
+
+    this.type = type ?? 'software-catalog';
+    this.locationTemplate =
+      locationTemplate || '/catalog/:namespace/:kind/:name';
+    this.filter = filter;
+    this.batchSize = batchSize || 500;
+    this.catalogClient =
+      catalogClient || new CatalogClient({ discoveryApi: discovery });
+    this.tokenManager = tokenManager;
+    this.entityTransformer =
+      entityTransformer ?? new DefaultCatalogCollatorEntityTransformer();
+  }
+
+  async getCollator(): Promise<Readable> {
+    return Readable.from(this.execute());
+  }
+
+  private async *execute(): AsyncGenerator<CatalogEntityDocument> {
+    const { token } = await this.tokenManager.getToken();
+    let entitiesRetrieved = 0;
+    let moreEntitiesToGet = true;
+
+    // Offset/limit pagination is used on the Catalog Client in order to
+    // limit (and allow some control over) memory used by the search backend
+    // at index-time.
+    while (moreEntitiesToGet) {
+      const entities = (
+        await this.catalogClient.getEntities(
+          {
+            filter: this.filter,
+            limit: this.batchSize,
+            offset: entitiesRetrieved,
+          },
+          { token },
+        )
+      ).items;
+
+      // Control looping through entity batches.
+      moreEntitiesToGet = entities.length === this.batchSize;
+      entitiesRetrieved += entities.length;
+
+      for (const entity of entities) {
+        yield this.entityTransformer.transform(entity, this.locationTemplate);
+      }
+    }
   }
 }

plugins/catalog-backend/src/search/index.ts

@@ -16,11 +16,7 @@
 
 export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory';
 export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory';
-export { CatalogCollatorFactory } from './CatalogCollatorFactory';
-export type {
-  CatalogCollatorFactoryOptions,
-  CatalogCollatorFactoryCreateOptions,
-} from './CatalogCollatorFactory';
+export type { CatalogCollatorEntityTransformer } from './CatalogCollatorEntityTransformer';
 
 /**
  * todo(backstage/techdocs-core): stop exporting this in a future release.