catalog filter extension point

Signed-off-by: Bond Yan <bondy@spotify.com>
2025-02-10 10:52:00 -05:00
parent 0934888547
commit d32bdc47fa
5 changed files with 138 additions and 73 deletions
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-search-backend-module-techdocs': patch
+---
+
+Added an extension point that allows for custom entity filtering during document collation.
@@ -72,6 +72,15 @@ export type TechDocsCollatorDocumentTransformer = (
  >
 >;

+// @public (undocumented)
+export interface TechDocsCollatorEntityFilterExtensionPoint {
+  // (undocumented)
+  setEntityFilter(filterFunction: (entities: Entity[]) => Entity[]): void;
+}
+
+// @public
+export const techDocsCollatorEntityFilterExtensionPoint: ExtensionPoint<TechDocsCollatorEntityFilterExtensionPoint>;
+
 // @public (undocumented)
 export type TechDocsCollatorEntityTransformer = (
  entity: Entity,
@@ -103,5 +112,6 @@ export type TechDocsCollatorFactoryOptions = {
  legacyPathCasing?: boolean;
  entityTransformer?: TechDocsCollatorEntityTransformer;
  documentTransformer?: TechDocsCollatorDocumentTransformer;
+  entityFilter?: (entity: Entity[]) => Entity[];
 };
 ```
@@ -193,6 +193,18 @@ describe('DefaultTechDocsCollatorFactory', () => {
      });
    });

+    it('should filter catalog entities when a custom filter is set', async () => {
+      factory = DefaultTechDocsCollatorFactory.fromConfig(config, {
+        ...options,
+        entityFilter: entities =>
+          entities.filter(entity => entity.kind !== 'Component'),
+      });
+      collator = await factory.getCollator();
+      const pipeline = TestPipeline.fromCollator(collator);
+      const { documents } = await pipeline.execute();
+      expect(documents).toHaveLength(0);
+    });
+
    it('paginates through catalog entities using batchSize', async () => {
      // A parallelismLimit of 1 is a catalog limit of 50 per request. Code
      // above in the /entities handler ensures valid entities are only
@@ -68,6 +68,7 @@ export type TechDocsCollatorFactoryOptions = {
  legacyPathCasing?: boolean;
  entityTransformer?: TechDocsCollatorEntityTransformer;
  documentTransformer?: TechDocsCollatorDocumentTransformer;
+  entityFilter?: (entity: Entity[]) => Entity[];
 };

 type EntityInfo = {
@@ -97,6 +98,7 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
  private readonly legacyPathCasing: boolean;
  private entityTransformer: TechDocsCollatorEntityTransformer;
  private documentTransformer: TechDocsCollatorDocumentTransformer;
+  private entityFilter: Function | undefined;

  private constructor(options: TechDocsCollatorFactoryOptions) {
    this.discovery = options.discovery;
@@ -110,6 +112,7 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
    this.legacyPathCasing = options.legacyPathCasing ?? false;
    this.entityTransformer = options.entityTransformer ?? (() => ({}));
    this.documentTransformer = options.documentTransformer ?? (() => ({}));
+    this.entityFilter = options.entityFilter;

    this.auth = createLegacyAuthAdapters({
      auth: options.auth,
@@ -177,81 +180,83 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
      moreEntitiesToGet = entities.length === batchSize;
      entitiesRetrieved += entities.length;

-      const docPromises = entities
-        .filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref'])
-        .map((entity: Entity) =>
-          limit(async (): Promise<TechDocsDocument[]> => {
-            const entityInfo =
-              DefaultTechDocsCollatorFactory.handleEntityInfoCasing(
-                this.legacyPathCasing,
+      const filteredEntities = this.entityFilter
+        ? this.entityFilter(entities)
+        : this.defaultFilteringFunction(entities);
+
+      const docPromises = filteredEntities.map((entity: Entity) =>
+        limit(async (): Promise<TechDocsDocument[]> => {
+          const entityInfo =
+            DefaultTechDocsCollatorFactory.handleEntityInfoCasing(
+              this.legacyPathCasing,
+              {
+                kind: entity.kind,
+                namespace: entity.metadata.namespace || 'default',
+                name: entity.metadata.name,
+              },
+            );
+
+          try {
+            const { token: techdocsToken } =
+              await this.auth.getPluginRequestToken({
+                onBehalfOf: await this.auth.getOwnServiceCredentials(),
+                targetPluginId: 'techdocs',
+              });
+
+            const searchIndexResponse = await fetch(
+              DefaultTechDocsCollatorFactory.constructDocsIndexUrl(
+                techDocsBaseUrl,
+                entityInfo,
+              ),
+              {
+                headers: {
+                  Authorization: `Bearer ${techdocsToken}`,
+                },
+              },
+            );
+
+            // todo(@backstage/techdocs-core): remove Promise.race() when node-fetch is 3.x+
+            // workaround for fetch().json() hanging in node-fetch@2.x.x, fixed in 3.x.x
+            // https://github.com/node-fetch/node-fetch/issues/665
+            const searchIndex = await Promise.race([
+              searchIndexResponse.json(),
+              new Promise((_resolve, reject) => {
+                setTimeout(() => {
+                  reject('Could not parse JSON in 5 seconds.');
+                }, 5000);
+              }),
+            ]);
+
+            return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({
+              ...defaultTechDocsCollatorEntityTransformer(entity),
+              ...defaultTechDocsCollatorDocumentTransformer(doc),
+              ...this.entityTransformer(entity),
+              ...this.documentTransformer(doc),
+              location: this.applyArgsToFormat(
+                this.locationTemplate || '/docs/:namespace/:kind/:name/:path',
                {
-                  kind: entity.kind,
-                  namespace: entity.metadata.namespace || 'default',
-                  name: entity.metadata.name,
+                  ...entityInfo,
+                  path: doc.location,
                },
-              );
-
-            try {
-              const { token: techdocsToken } =
-                await this.auth.getPluginRequestToken({
-                  onBehalfOf: await this.auth.getOwnServiceCredentials(),
-                  targetPluginId: 'techdocs',
-                });
-
-              const searchIndexResponse = await fetch(
-                DefaultTechDocsCollatorFactory.constructDocsIndexUrl(
-                  techDocsBaseUrl,
-                  entityInfo,
-                ),
-                {
-                  headers: {
-                    Authorization: `Bearer ${techdocsToken}`,
-                  },
-                },
-              );
-
-              // todo(@backstage/techdocs-core): remove Promise.race() when node-fetch is 3.x+
-              // workaround for fetch().json() hanging in node-fetch@2.x.x, fixed in 3.x.x
-              // https://github.com/node-fetch/node-fetch/issues/665
-              const searchIndex = await Promise.race([
-                searchIndexResponse.json(),
-                new Promise((_resolve, reject) => {
-                  setTimeout(() => {
-                    reject('Could not parse JSON in 5 seconds.');
-                  }, 5000);
-                }),
-              ]);
-
-              return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({
-                ...defaultTechDocsCollatorEntityTransformer(entity),
-                ...defaultTechDocsCollatorDocumentTransformer(doc),
-                ...this.entityTransformer(entity),
-                ...this.documentTransformer(doc),
-                location: this.applyArgsToFormat(
-                  this.locationTemplate || '/docs/:namespace/:kind/:name/:path',
-                  {
-                    ...entityInfo,
-                    path: doc.location,
-                  },
-                ),
-                ...entityInfo,
-                entityTitle: entity.metadata.title,
-                componentType: entity.spec?.type?.toString() || 'other',
-                lifecycle: (entity.spec?.lifecycle as string) || '',
-                owner: getSimpleEntityOwnerString(entity),
-                authorization: {
-                  resourceRef: stringifyEntityRef(entity),
-                },
-              }));
-            } catch (e) {
-              this.logger.debug(
-                `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`,
-                e,
-              );
-              return [];
-            }
-          }),
-        );
+              ),
+              ...entityInfo,
+              entityTitle: entity.metadata.title,
+              componentType: entity.spec?.type?.toString() || 'other',
+              lifecycle: (entity.spec?.lifecycle as string) || '',
+              owner: getSimpleEntityOwnerString(entity),
+              authorization: {
+                resourceRef: stringifyEntityRef(entity),
+              },
+            }));
+          } catch (e) {
+            this.logger.debug(
+              `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`,
+              e,
+            );
+            return [];
+          }
+        }),
+      );
      yield* (await Promise.all(docPromises)).flat();
    }
  }
@@ -267,6 +272,12 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
    return formatted;
  }

+  private defaultFilteringFunction(entities: Entity[]): Entity[] {
+    return entities.filter(
+      entity => entity.metadata?.annotations?.['backstage.io/techdocs-ref'],
+    );
+  }
+
  private static constructDocsIndexUrl(
    techDocsBaseUrl: string,
    entityInfo: { kind: string; namespace: string; name: string },
@@ -25,6 +25,7 @@ import {
  createExtensionPoint,
  readSchedulerServiceTaskScheduleDefinitionFromConfig,
 } from '@backstage/backend-plugin-api';
+import { Entity } from '@backstage/catalog-model';
 import { catalogServiceRef } from '@backstage/plugin-catalog-node/alpha';
 import {
  DefaultTechDocsCollatorFactory,
@@ -51,6 +52,21 @@ export const techdocsCollatorEntityTransformerExtensionPoint =
    id: 'search.techdocsCollator.transformer',
  });

+/** @public */
+export interface TechDocsCollatorEntityFilterExtensionPoint {
+  setEntityFilter(filterFunction: (entities: Entity[]) => Entity[]): void;
+}
+
+/**
+ * Extension point used to filter the entities that the collator will use.
+ *
+ * @public
+ */
+export const techDocsCollatorEntityFilterExtensionPoint =
+  createExtensionPoint<TechDocsCollatorEntityFilterExtensionPoint>({
+    id: 'search.techdocsCollator.entityFilter',
+  });
+
 /**
 * @public
 * Search backend module for the TechDocs index.
@@ -61,6 +77,7 @@ export default createBackendModule({
  register(env) {
    let entityTransformer: TechDocsCollatorEntityTransformer | undefined;
    let documentTransformer: TechDocsCollatorDocumentTransformer | undefined;
+    let entityFilter: ((e: Entity[]) => Entity[]) | undefined;

    env.registerExtensionPoint(
      techdocsCollatorEntityTransformerExtensionPoint,
@@ -84,6 +101,15 @@ export default createBackendModule({
      },
    );

+    env.registerExtensionPoint(techDocsCollatorEntityFilterExtensionPoint, {
+      setEntityFilter(newEntityFilter) {
+        if (entityFilter) {
+          throw new Error('TechDocs entity filters may only be set once');
+        }
+        entityFilter = newEntityFilter;
+      },
+    });
+
    env.registerInit({
      deps: {
        config: coreServices.rootConfig,
@@ -127,6 +153,7 @@ export default createBackendModule({
            catalogClient: catalog,
            entityTransformer,
            documentTransformer,
+            entityFilter,
          }),
        });
      },