diff --git a/.changeset/blue-camels-fry.md b/.changeset/blue-camels-fry.md new file mode 100644 index 0000000000..1f24a62b5b --- /dev/null +++ b/.changeset/blue-camels-fry.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-techdocs': patch +--- + +Added an extension point that allows for custom entity filtering during document collation. diff --git a/docs/features/search/collators.md b/docs/features/search/collators.md index 834a14437a..d87258f65b 100644 --- a/docs/features/search/collators.md +++ b/docs/features/search/collators.md @@ -134,6 +134,38 @@ search: timeout: { minutes: 3 } ``` +### Filtering through the catalog collator + +The TechDocs collator by default filters through catalog entities where the annotation `metadata.annotations.backstage.io/techdocs-ref` exists. If you wish to further filter out entities, there are two ways to do so through the `techDocsCollatorEntityFilterExtensionPoint`. + +```typescript +export const exampleCustomCatalogFiltering = createBackendModule({ + pluginId: 'search', + moduleId: 'search-techdocs-collator-entity-filter', + register(reg) { + reg.registerInit({ + deps: { + customCollatorFilter: techDocsCollatorEntityFilterExtensionPoint, + }, + async init({ customCollatorFilter }) { + /* filtering by catalog params */ + customCollatorFilter.setCustomCatalogApiFilters([ + { kind: ['API', 'Component', ...] }, + { metadata: ['...more filters'] }, + ]); + + /* filtering by a custom function */ + customCollatorFilter.setEntityFilterFunction((entities: Entity[]) => + entities.filter( + entity => entity.metadata?.annotations?.abc === 'xyz', + ), + ); + }, + }); + }, +}); +``` + ## Community Collators Here are some of the known Search Collators available in from the Backstage Community: diff --git a/plugins/search-backend-module-techdocs/report.api.md b/plugins/search-backend-module-techdocs/report.api.md index fadcd5578f..ee8999be13 100644 --- a/plugins/search-backend-module-techdocs/report.api.md +++ b/plugins/search-backend-module-techdocs/report.api.md @@ -10,6 +10,7 @@ import { Config } from '@backstage/config'; import { DiscoveryService } from '@backstage/backend-plugin-api'; import { DocumentCollatorFactory } from '@backstage/plugin-search-common'; import { Entity } from '@backstage/catalog-model'; +import { EntityFilterQuery } from '@backstage/catalog-client'; import { ExtensionPoint } from '@backstage/backend-plugin-api'; import { HttpAuthService } from '@backstage/backend-plugin-api'; import { LoggerService } from '@backstage/backend-plugin-api'; @@ -70,6 +71,19 @@ export type TechDocsCollatorDocumentTransformer = ( > >; +// @public (undocumented) +export interface TechDocsCollatorEntityFilterExtensionPoint { + // (undocumented) + setCustomCatalogApiFilters(apiFilters: EntityFilterQuery): void; + // (undocumented) + setEntityFilterFunction( + filterFunction: (entities: Entity[]) => Entity[], + ): void; +} + +// @public +export const techDocsCollatorEntityFilterExtensionPoint: ExtensionPoint; + // @public (undocumented) export type TechDocsCollatorEntityTransformer = ( entity: Entity, @@ -101,5 +115,7 @@ export type TechDocsCollatorFactoryOptions = { legacyPathCasing?: boolean; entityTransformer?: TechDocsCollatorEntityTransformer; documentTransformer?: TechDocsCollatorDocumentTransformer; + entityFilterFunction?: (entity: Entity[]) => Entity[]; + customCatalogApiFilters?: EntityFilterQuery; }; ``` diff --git a/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.test.ts b/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.test.ts index 2b8ec17b0d..3aff8cbbb7 100644 --- a/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.test.ts +++ b/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.test.ts @@ -193,6 +193,18 @@ describe('DefaultTechDocsCollatorFactory', () => { }); }); + it('should filter catalog entities when a custom filter is set', async () => { + factory = DefaultTechDocsCollatorFactory.fromConfig(config, { + ...options, + entityFilterFunction: entities => + entities.filter(entity => entity.kind !== 'Component'), + }); + collator = await factory.getCollator(); + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + expect(documents).toHaveLength(0); + }); + it('paginates through catalog entities using batchSize', async () => { // A parallelismLimit of 1 is a catalog limit of 50 per request. Code // above in the /entities handler ensures valid entities are only diff --git a/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.ts b/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.ts index 3a586fd306..5331cd9105 100644 --- a/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.ts +++ b/plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.ts @@ -22,6 +22,7 @@ import { CATALOG_FILTER_EXISTS, CatalogApi, CatalogClient, + EntityFilterQuery, } from '@backstage/catalog-client'; import { Entity, @@ -68,6 +69,8 @@ export type TechDocsCollatorFactoryOptions = { legacyPathCasing?: boolean; entityTransformer?: TechDocsCollatorEntityTransformer; documentTransformer?: TechDocsCollatorDocumentTransformer; + entityFilterFunction?: (entity: Entity[]) => Entity[]; + customCatalogApiFilters?: EntityFilterQuery; }; type EntityInfo = { @@ -97,6 +100,8 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { private readonly legacyPathCasing: boolean; private entityTransformer: TechDocsCollatorEntityTransformer; private documentTransformer: TechDocsCollatorDocumentTransformer; + private entityFilterFunction: Function | undefined; + private customCatalogApiFilters: EntityFilterQuery | undefined; private constructor(options: TechDocsCollatorFactoryOptions) { this.discovery = options.discovery; @@ -110,6 +115,8 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { this.legacyPathCasing = options.legacyPathCasing ?? false; this.entityTransformer = options.entityTransformer ?? (() => ({})); this.documentTransformer = options.documentTransformer ?? (() => ({})); + this.entityFilterFunction = options.entityFilterFunction; + this.customCatalogApiFilters = options.customCatalogApiFilters; this.auth = createLegacyAuthAdapters({ auth: options.auth, @@ -165,6 +172,7 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { filter: { 'metadata.annotations.backstage.io/techdocs-ref': CATALOG_FILTER_EXISTS, + ...this.customCatalogApiFilters, }, limit: batchSize, offset: entitiesRetrieved, @@ -177,81 +185,71 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { moreEntitiesToGet = entities.length === batchSize; entitiesRetrieved += entities.length; - const docPromises = entities - .filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref']) - .map((entity: Entity) => - limit(async (): Promise => { - const entityInfo = - DefaultTechDocsCollatorFactory.handleEntityInfoCasing( - this.legacyPathCasing, + const filteredEntities = this.entityFilterFunction + ? this.entityFilterFunction(entities) + : this.defaultFilteringFunction(entities); + + const docPromises = filteredEntities.map((entity: Entity) => + limit(async (): Promise => { + const entityInfo = + DefaultTechDocsCollatorFactory.handleEntityInfoCasing( + this.legacyPathCasing, + { + kind: entity.kind, + namespace: entity.metadata.namespace || 'default', + name: entity.metadata.name, + }, + ); + + try { + const { token: techdocsToken } = + await this.auth.getPluginRequestToken({ + onBehalfOf: await this.auth.getOwnServiceCredentials(), + targetPluginId: 'techdocs', + }); + + const searchIndex = await fetch( + DefaultTechDocsCollatorFactory.constructDocsIndexUrl( + techDocsBaseUrl, + entityInfo, + ), + { + headers: { + Authorization: `Bearer ${techdocsToken}`, + }, + }, + ).then(res => res.json()); + + return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({ + ...defaultTechDocsCollatorEntityTransformer(entity), + ...defaultTechDocsCollatorDocumentTransformer(doc), + ...this.entityTransformer(entity), + ...this.documentTransformer(doc), + location: this.applyArgsToFormat( + this.locationTemplate || '/docs/:namespace/:kind/:name/:path', { - kind: entity.kind, - namespace: entity.metadata.namespace || 'default', - name: entity.metadata.name, + ...entityInfo, + path: doc.location, }, - ); - - try { - const { token: techdocsToken } = - await this.auth.getPluginRequestToken({ - onBehalfOf: await this.auth.getOwnServiceCredentials(), - targetPluginId: 'techdocs', - }); - - const searchIndexResponse = await fetch( - DefaultTechDocsCollatorFactory.constructDocsIndexUrl( - techDocsBaseUrl, - entityInfo, - ), - { - headers: { - Authorization: `Bearer ${techdocsToken}`, - }, - }, - ); - - // todo(@backstage/techdocs-core): remove Promise.race() when node-fetch is 3.x+ - // workaround for fetch().json() hanging in node-fetch@2.x.x, fixed in 3.x.x - // https://github.com/node-fetch/node-fetch/issues/665 - const searchIndex = await Promise.race([ - searchIndexResponse.json(), - new Promise((_resolve, reject) => { - setTimeout(() => { - reject('Could not parse JSON in 5 seconds.'); - }, 5000); - }), - ]); - - return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({ - ...defaultTechDocsCollatorEntityTransformer(entity), - ...defaultTechDocsCollatorDocumentTransformer(doc), - ...this.entityTransformer(entity), - ...this.documentTransformer(doc), - location: this.applyArgsToFormat( - this.locationTemplate || '/docs/:namespace/:kind/:name/:path', - { - ...entityInfo, - path: doc.location, - }, - ), - ...entityInfo, - entityTitle: entity.metadata.title, - componentType: entity.spec?.type?.toString() || 'other', - lifecycle: (entity.spec?.lifecycle as string) || '', - owner: getSimpleEntityOwnerString(entity), - authorization: { - resourceRef: stringifyEntityRef(entity), - }, - })); - } catch (e) { - this.logger.debug( - `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`, - e, - ); - return []; - } - }), - ); + ), + ...entityInfo, + entityTitle: entity.metadata.title, + componentType: entity.spec?.type?.toString() || 'other', + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: getSimpleEntityOwnerString(entity), + authorization: { + resourceRef: stringifyEntityRef(entity), + }, + })); + } catch (e) { + this.logger.debug( + `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`, + e, + ); + return []; + } + }), + ); yield* (await Promise.all(docPromises)).flat(); } } @@ -267,6 +265,12 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { return formatted; } + private defaultFilteringFunction(entities: Entity[]): Entity[] { + return entities.filter( + entity => entity.metadata?.annotations?.['backstage.io/techdocs-ref'], + ); + } + private static constructDocsIndexUrl( techDocsBaseUrl: string, entityInfo: { kind: string; namespace: string; name: string }, diff --git a/plugins/search-backend-module-techdocs/src/module.ts b/plugins/search-backend-module-techdocs/src/module.ts index 0114b88c2d..2888b9527b 100644 --- a/plugins/search-backend-module-techdocs/src/module.ts +++ b/plugins/search-backend-module-techdocs/src/module.ts @@ -25,6 +25,8 @@ import { createExtensionPoint, readSchedulerServiceTaskScheduleDefinitionFromConfig, } from '@backstage/backend-plugin-api'; +import { EntityFilterQuery } from '@backstage/catalog-client'; +import { Entity } from '@backstage/catalog-model'; import { catalogServiceRef } from '@backstage/plugin-catalog-node/alpha'; import { DefaultTechDocsCollatorFactory, @@ -51,6 +53,24 @@ export const techdocsCollatorEntityTransformerExtensionPoint = id: 'search.techdocsCollator.transformer', }); +/** @public */ +export interface TechDocsCollatorEntityFilterExtensionPoint { + setEntityFilterFunction( + filterFunction: (entities: Entity[]) => Entity[], + ): void; + setCustomCatalogApiFilters(apiFilters: EntityFilterQuery): void; +} + +/** + * Extension point used to filter the entities that the collator will use. + * + * @public + */ +export const techDocsCollatorEntityFilterExtensionPoint = + createExtensionPoint({ + id: 'search.techdocsCollator.entityFilter', + }); + /** * @public * Search backend module for the TechDocs index. @@ -61,6 +81,8 @@ export default createBackendModule({ register(env) { let entityTransformer: TechDocsCollatorEntityTransformer | undefined; let documentTransformer: TechDocsCollatorDocumentTransformer | undefined; + let entityFilterFunction: ((e: Entity[]) => Entity[]) | undefined; + let customCatalogApiFilters: EntityFilterQuery | undefined; env.registerExtensionPoint( techdocsCollatorEntityTransformerExtensionPoint, @@ -84,6 +106,25 @@ export default createBackendModule({ }, ); + env.registerExtensionPoint(techDocsCollatorEntityFilterExtensionPoint, { + setEntityFilterFunction(newEntityFilterFunction) { + if (entityFilterFunction) { + throw new Error( + 'TechDocs entity filter functions may only be set once', + ); + } + entityFilterFunction = newEntityFilterFunction; + }, + setCustomCatalogApiFilters(newCatalogApiFilters) { + if (customCatalogApiFilters) { + throw new Error( + 'TechDocs catalog entity filters may only be set once', + ); + } + customCatalogApiFilters = newCatalogApiFilters; + }, + }); + env.registerInit({ deps: { config: coreServices.rootConfig, @@ -127,6 +168,8 @@ export default createBackendModule({ catalogClient: catalog, entityTransformer, documentTransformer, + customCatalogApiFilters, + entityFilterFunction, }), }); },