diff --git a/.changeset/blue-camels-fry.md b/.changeset/blue-camels-fry.md new file mode 100644 index 0000000000..1f24a62b5b --- /dev/null +++ b/.changeset/blue-camels-fry.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-techdocs': patch +--- + +Added an extension point that allows for custom entity filtering during document collation. diff --git a/plugins/search-backend-module-techdocs/report.api.md b/plugins/search-backend-module-techdocs/report.api.md index ef041e6900..1365af4545 100644 --- a/plugins/search-backend-module-techdocs/report.api.md +++ b/plugins/search-backend-module-techdocs/report.api.md @@ -72,6 +72,15 @@ export type TechDocsCollatorDocumentTransformer = ( > >; +// @public (undocumented) +export interface TechDocsCollatorEntityFilterExtensionPoint { + // (undocumented) + setEntityFilter(filterFunction: (entities: Entity[]) => Entity[]): void; +} + +// @public +export const techDocsCollatorEntityFilterExtensionPoint: ExtensionPoint; + // @public (undocumented) export type TechDocsCollatorEntityTransformer = ( entity: Entity, @@ -103,5 +112,6 @@ export type TechDocsCollatorFactoryOptions = { legacyPathCasing?: boolean; entityTransformer?: TechDocsCollatorEntityTransformer; documentTransformer?: TechDocsCollatorDocumentTransformer; + entityFilter?: (entity: Entity[]) => Entity[]; }; ``` diff --git a/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.test.ts b/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.test.ts index 2b8ec17b0d..b32cf7bd7a 100644 --- a/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.test.ts +++ b/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.test.ts @@ -193,6 +193,18 @@ describe('DefaultTechDocsCollatorFactory', () => { }); }); + it('should filter catalog entities when a custom filter is set', async () => { + factory = DefaultTechDocsCollatorFactory.fromConfig(config, { + ...options, + entityFilter: entities => + entities.filter(entity => entity.kind !== 'Component'), + }); + collator = await factory.getCollator(); + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + expect(documents).toHaveLength(0); + }); + it('paginates through catalog entities using batchSize', async () => { // A parallelismLimit of 1 is a catalog limit of 50 per request. Code // above in the /entities handler ensures valid entities are only diff --git a/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.ts b/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.ts index 3a586fd306..fb8393898a 100644 --- a/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.ts +++ b/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.ts @@ -68,6 +68,7 @@ export type TechDocsCollatorFactoryOptions = { legacyPathCasing?: boolean; entityTransformer?: TechDocsCollatorEntityTransformer; documentTransformer?: TechDocsCollatorDocumentTransformer; + entityFilter?: (entity: Entity[]) => Entity[]; }; type EntityInfo = { @@ -97,6 +98,7 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { private readonly legacyPathCasing: boolean; private entityTransformer: TechDocsCollatorEntityTransformer; private documentTransformer: TechDocsCollatorDocumentTransformer; + private entityFilter: Function | undefined; private constructor(options: TechDocsCollatorFactoryOptions) { this.discovery = options.discovery; @@ -110,6 +112,7 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { this.legacyPathCasing = options.legacyPathCasing ?? false; this.entityTransformer = options.entityTransformer ?? (() => ({})); this.documentTransformer = options.documentTransformer ?? (() => ({})); + this.entityFilter = options.entityFilter; this.auth = createLegacyAuthAdapters({ auth: options.auth, @@ -177,81 +180,83 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { moreEntitiesToGet = entities.length === batchSize; entitiesRetrieved += entities.length; - const docPromises = entities - .filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref']) - .map((entity: Entity) => - limit(async (): Promise => { - const entityInfo = - DefaultTechDocsCollatorFactory.handleEntityInfoCasing( - this.legacyPathCasing, + const filteredEntities = this.entityFilter + ? this.entityFilter(entities) + : this.defaultFilteringFunction(entities); + + const docPromises = filteredEntities.map((entity: Entity) => + limit(async (): Promise => { + const entityInfo = + DefaultTechDocsCollatorFactory.handleEntityInfoCasing( + this.legacyPathCasing, + { + kind: entity.kind, + namespace: entity.metadata.namespace || 'default', + name: entity.metadata.name, + }, + ); + + try { + const { token: techdocsToken } = + await this.auth.getPluginRequestToken({ + onBehalfOf: await this.auth.getOwnServiceCredentials(), + targetPluginId: 'techdocs', + }); + + const searchIndexResponse = await fetch( + DefaultTechDocsCollatorFactory.constructDocsIndexUrl( + techDocsBaseUrl, + entityInfo, + ), + { + headers: { + Authorization: `Bearer ${techdocsToken}`, + }, + }, + ); + + // todo(@backstage/techdocs-core): remove Promise.race() when node-fetch is 3.x+ + // workaround for fetch().json() hanging in node-fetch@2.x.x, fixed in 3.x.x + // https://github.com/node-fetch/node-fetch/issues/665 + const searchIndex = await Promise.race([ + searchIndexResponse.json(), + new Promise((_resolve, reject) => { + setTimeout(() => { + reject('Could not parse JSON in 5 seconds.'); + }, 5000); + }), + ]); + + return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({ + ...defaultTechDocsCollatorEntityTransformer(entity), + ...defaultTechDocsCollatorDocumentTransformer(doc), + ...this.entityTransformer(entity), + ...this.documentTransformer(doc), + location: this.applyArgsToFormat( + this.locationTemplate || '/docs/:namespace/:kind/:name/:path', { - kind: entity.kind, - namespace: entity.metadata.namespace || 'default', - name: entity.metadata.name, + ...entityInfo, + path: doc.location, }, - ); - - try { - const { token: techdocsToken } = - await this.auth.getPluginRequestToken({ - onBehalfOf: await this.auth.getOwnServiceCredentials(), - targetPluginId: 'techdocs', - }); - - const searchIndexResponse = await fetch( - DefaultTechDocsCollatorFactory.constructDocsIndexUrl( - techDocsBaseUrl, - entityInfo, - ), - { - headers: { - Authorization: `Bearer ${techdocsToken}`, - }, - }, - ); - - // todo(@backstage/techdocs-core): remove Promise.race() when node-fetch is 3.x+ - // workaround for fetch().json() hanging in node-fetch@2.x.x, fixed in 3.x.x - // https://github.com/node-fetch/node-fetch/issues/665 - const searchIndex = await Promise.race([ - searchIndexResponse.json(), - new Promise((_resolve, reject) => { - setTimeout(() => { - reject('Could not parse JSON in 5 seconds.'); - }, 5000); - }), - ]); - - return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({ - ...defaultTechDocsCollatorEntityTransformer(entity), - ...defaultTechDocsCollatorDocumentTransformer(doc), - ...this.entityTransformer(entity), - ...this.documentTransformer(doc), - location: this.applyArgsToFormat( - this.locationTemplate || '/docs/:namespace/:kind/:name/:path', - { - ...entityInfo, - path: doc.location, - }, - ), - ...entityInfo, - entityTitle: entity.metadata.title, - componentType: entity.spec?.type?.toString() || 'other', - lifecycle: (entity.spec?.lifecycle as string) || '', - owner: getSimpleEntityOwnerString(entity), - authorization: { - resourceRef: stringifyEntityRef(entity), - }, - })); - } catch (e) { - this.logger.debug( - `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`, - e, - ); - return []; - } - }), - ); + ), + ...entityInfo, + entityTitle: entity.metadata.title, + componentType: entity.spec?.type?.toString() || 'other', + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: getSimpleEntityOwnerString(entity), + authorization: { + resourceRef: stringifyEntityRef(entity), + }, + })); + } catch (e) { + this.logger.debug( + `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`, + e, + ); + return []; + } + }), + ); yield* (await Promise.all(docPromises)).flat(); } } @@ -267,6 +272,12 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { return formatted; } + private defaultFilteringFunction(entities: Entity[]): Entity[] { + return entities.filter( + entity => entity.metadata?.annotations?.['backstage.io/techdocs-ref'], + ); + } + private static constructDocsIndexUrl( techDocsBaseUrl: string, entityInfo: { kind: string; namespace: string; name: string }, diff --git a/plugins/search-backend-module-techdocs/src/module.ts b/plugins/search-backend-module-techdocs/src/module.ts index 0114b88c2d..68749a21ab 100644 --- a/plugins/search-backend-module-techdocs/src/module.ts +++ b/plugins/search-backend-module-techdocs/src/module.ts @@ -25,6 +25,7 @@ import { createExtensionPoint, readSchedulerServiceTaskScheduleDefinitionFromConfig, } from '@backstage/backend-plugin-api'; +import { Entity } from '@backstage/catalog-model'; import { catalogServiceRef } from '@backstage/plugin-catalog-node/alpha'; import { DefaultTechDocsCollatorFactory, @@ -51,6 +52,21 @@ export const techdocsCollatorEntityTransformerExtensionPoint = id: 'search.techdocsCollator.transformer', }); +/** @public */ +export interface TechDocsCollatorEntityFilterExtensionPoint { + setEntityFilter(filterFunction: (entities: Entity[]) => Entity[]): void; +} + +/** + * Extension point used to filter the entities that the collator will use. + * + * @public + */ +export const techDocsCollatorEntityFilterExtensionPoint = + createExtensionPoint({ + id: 'search.techdocsCollator.entityFilter', + }); + /** * @public * Search backend module for the TechDocs index. @@ -61,6 +77,7 @@ export default createBackendModule({ register(env) { let entityTransformer: TechDocsCollatorEntityTransformer | undefined; let documentTransformer: TechDocsCollatorDocumentTransformer | undefined; + let entityFilter: ((e: Entity[]) => Entity[]) | undefined; env.registerExtensionPoint( techdocsCollatorEntityTransformerExtensionPoint, @@ -84,6 +101,15 @@ export default createBackendModule({ }, ); + env.registerExtensionPoint(techDocsCollatorEntityFilterExtensionPoint, { + setEntityFilter(newEntityFilter) { + if (entityFilter) { + throw new Error('TechDocs entity filters may only be set once'); + } + entityFilter = newEntityFilter; + }, + }); + env.registerInit({ deps: { config: coreServices.rootConfig, @@ -127,6 +153,7 @@ export default createBackendModule({ catalogClient: catalog, entityTransformer, documentTransformer, + entityFilter, }), }); },