paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 27 Packages Projects Releases Wiki Activity

Merge pull request #28795 from backstage/techdocs-collator-filter-extension

Browse Source 
Custom catalog filtering within TechDocs collator
This commit is contained in:
bond yan
2025-03-13 13:24:01 -04:00
committed by GitHub
parent 04b7445c35 e7a3b2ed2e
commit ba409c3d54
6 changed files with 185 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/blue-camels-fry.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-search-backend-module-techdocs': patch
---
Added an extension point that allows for custom entity filtering during document collation.
docs/features/search/collators.md
+32
View File
@@ -134,6 +134,38 @@ search:
timeout: { minutes: 3 }
```
### Filtering through the catalog collator
The TechDocs collator by default filters through catalog entities where the annotation `metadata.annotations.backstage.io/techdocs-ref` exists. If you wish to further filter out entities, there are two ways to do so through the `techDocsCollatorEntityFilterExtensionPoint`.
```typescript
export const exampleCustomCatalogFiltering = createBackendModule({
pluginId: 'search',
moduleId: 'search-techdocs-collator-entity-filter',
register(reg) {
reg.registerInit({
deps: {
customCollatorFilter: techDocsCollatorEntityFilterExtensionPoint,
},
async init({ customCollatorFilter }) {
/* filtering by catalog params */
customCollatorFilter.setCustomCatalogApiFilters([
{ kind: ['API', 'Component', ...] },
{ metadata: ['...more filters'] },
]);
/* filtering by a custom function */
customCollatorFilter.setEntityFilterFunction((entities: Entity[]) =>
entities.filter(
entity => entity.metadata?.annotations?.abc === 'xyz',
),
);
},
});
},
});
```
## Community Collators
Here are some of the known Search Collators available in from the Backstage Community:
plugins/search-backend-module-techdocs/report.api.md
+16
View File
@@ -10,6 +10,7 @@ import { Config } from '@backstage/config';
import { DiscoveryService } from '@backstage/backend-plugin-api';
import { DocumentCollatorFactory } from '@backstage/plugin-search-common';
import { Entity } from '@backstage/catalog-model';
import { EntityFilterQuery } from '@backstage/catalog-client';
import { ExtensionPoint } from '@backstage/backend-plugin-api';
import { HttpAuthService } from '@backstage/backend-plugin-api';
import { LoggerService } from '@backstage/backend-plugin-api';
@@ -70,6 +71,19 @@ export type TechDocsCollatorDocumentTransformer = (
>
>;
// @public (undocumented)
export interface TechDocsCollatorEntityFilterExtensionPoint {
// (undocumented)
setCustomCatalogApiFilters(apiFilters: EntityFilterQuery): void;
// (undocumented)
setEntityFilterFunction(
filterFunction: (entities: Entity[]) => Entity[],
): void;
}
// @public
export const techDocsCollatorEntityFilterExtensionPoint: ExtensionPoint<TechDocsCollatorEntityFilterExtensionPoint>;
// @public (undocumented)
export type TechDocsCollatorEntityTransformer = (
entity: Entity,
@@ -101,5 +115,7 @@ export type TechDocsCollatorFactoryOptions = {
legacyPathCasing?: boolean;
entityTransformer?: TechDocsCollatorEntityTransformer;
documentTransformer?: TechDocsCollatorDocumentTransformer;
entityFilterFunction?: (entity: Entity[]) => Entity[];
customCatalogApiFilters?: EntityFilterQuery;
};
```
plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.test.ts
+12
View File
@@ -193,6 +193,18 @@ describe('DefaultTechDocsCollatorFactory', () => {
});
});
it('should filter catalog entities when a custom filter is set', async () => {
factory = DefaultTechDocsCollatorFactory.fromConfig(config, {
...options,
entityFilterFunction: entities =>
entities.filter(entity => entity.kind !== 'Component'),
});
collator = await factory.getCollator();
const pipeline = TestPipeline.fromCollator(collator);
const { documents } = await pipeline.execute();
expect(documents).toHaveLength(0);
});
it('paginates through catalog entities using batchSize', async () => {
// A parallelismLimit of 1 is a catalog limit of 50 per request. Code
// above in the /entities handler ensures valid entities are only
plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.ts
+77 -73
View File
@@ -22,6 +22,7 @@ import {
CATALOG_FILTER_EXISTS,
CatalogApi,
CatalogClient,
EntityFilterQuery,
} from '@backstage/catalog-client';
import {
Entity,
@@ -68,6 +69,8 @@ export type TechDocsCollatorFactoryOptions = {
legacyPathCasing?: boolean;
entityTransformer?: TechDocsCollatorEntityTransformer;
documentTransformer?: TechDocsCollatorDocumentTransformer;
entityFilterFunction?: (entity: Entity[]) => Entity[];
customCatalogApiFilters?: EntityFilterQuery;
};
type EntityInfo = {
@@ -97,6 +100,8 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
private readonly legacyPathCasing: boolean;
private entityTransformer: TechDocsCollatorEntityTransformer;
private documentTransformer: TechDocsCollatorDocumentTransformer;
private entityFilterFunction: Function | undefined;
private customCatalogApiFilters: EntityFilterQuery | undefined;
private constructor(options: TechDocsCollatorFactoryOptions) {
this.discovery = options.discovery;
@@ -110,6 +115,8 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
this.legacyPathCasing = options.legacyPathCasing ?? false;
this.entityTransformer = options.entityTransformer ?? (() => ({}));
this.documentTransformer = options.documentTransformer ?? (() => ({}));
this.entityFilterFunction = options.entityFilterFunction;
this.customCatalogApiFilters = options.customCatalogApiFilters;
this.auth = createLegacyAuthAdapters({
auth: options.auth,
@@ -165,6 +172,7 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
filter: {
'metadata.annotations.backstage.io/techdocs-ref':
CATALOG_FILTER_EXISTS,
...this.customCatalogApiFilters,
},
limit: batchSize,
offset: entitiesRetrieved,
@@ -177,81 +185,71 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
moreEntitiesToGet = entities.length === batchSize;
entitiesRetrieved += entities.length;
const docPromises = entities
.filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref'])
.map((entity: Entity) =>
limit(async (): Promise<TechDocsDocument[]> => {
const entityInfo =
DefaultTechDocsCollatorFactory.handleEntityInfoCasing(
this.legacyPathCasing,
const filteredEntities = this.entityFilterFunction
? this.entityFilterFunction(entities)
: this.defaultFilteringFunction(entities);
const docPromises = filteredEntities.map((entity: Entity) =>
limit(async (): Promise<TechDocsDocument[]> => {
const entityInfo =
DefaultTechDocsCollatorFactory.handleEntityInfoCasing(
this.legacyPathCasing,
{
kind: entity.kind,
namespace: entity.metadata.namespace || 'default',
name: entity.metadata.name,
},
);
try {
const { token: techdocsToken } =
await this.auth.getPluginRequestToken({
onBehalfOf: await this.auth.getOwnServiceCredentials(),
targetPluginId: 'techdocs',
});
const searchIndex = await fetch(
DefaultTechDocsCollatorFactory.constructDocsIndexUrl(
techDocsBaseUrl,
entityInfo,
),
{
headers: {
Authorization: `Bearer ${techdocsToken}`,
},
},
).then(res => res.json());
return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({
...defaultTechDocsCollatorEntityTransformer(entity),
...defaultTechDocsCollatorDocumentTransformer(doc),
...this.entityTransformer(entity),
...this.documentTransformer(doc),
location: this.applyArgsToFormat(
this.locationTemplate || '/docs/:namespace/:kind/:name/:path',
{
kind: entity.kind,
namespace: entity.metadata.namespace || 'default',
name: entity.metadata.name,
...entityInfo,
path: doc.location,
},
);
try {
const { token: techdocsToken } =
await this.auth.getPluginRequestToken({
onBehalfOf: await this.auth.getOwnServiceCredentials(),
targetPluginId: 'techdocs',
});
const searchIndexResponse = await fetch(
DefaultTechDocsCollatorFactory.constructDocsIndexUrl(
techDocsBaseUrl,
entityInfo,
),
{
headers: {
Authorization: `Bearer ${techdocsToken}`,
},
},
);
// todo(@backstage/techdocs-core): remove Promise.race() when node-fetch is 3.x+
// workaround for fetch().json() hanging in node-fetch@2.x.x, fixed in 3.x.x
// https://github.com/node-fetch/node-fetch/issues/665
const searchIndex = await Promise.race([
searchIndexResponse.json(),
new Promise((_resolve, reject) => {
setTimeout(() => {
reject('Could not parse JSON in 5 seconds.');
}, 5000);
}),
]);
return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({
...defaultTechDocsCollatorEntityTransformer(entity),
...defaultTechDocsCollatorDocumentTransformer(doc),
...this.entityTransformer(entity),
...this.documentTransformer(doc),
location: this.applyArgsToFormat(
this.locationTemplate || '/docs/:namespace/:kind/:name/:path',
{
...entityInfo,
path: doc.location,
},
),
...entityInfo,
entityTitle: entity.metadata.title,
componentType: entity.spec?.type?.toString() || 'other',
lifecycle: (entity.spec?.lifecycle as string) || '',
owner: getSimpleEntityOwnerString(entity),
authorization: {
resourceRef: stringifyEntityRef(entity),
},
}));
} catch (e) {
this.logger.debug(
`Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`,
e,
);
return [];
}
}),
);
),
...entityInfo,
entityTitle: entity.metadata.title,
componentType: entity.spec?.type?.toString() || 'other',
lifecycle: (entity.spec?.lifecycle as string) || '',
owner: getSimpleEntityOwnerString(entity),
authorization: {
resourceRef: stringifyEntityRef(entity),
},
}));
} catch (e) {
this.logger.debug(
`Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`,
e,
);
return [];
}
}),
);
yield* (await Promise.all(docPromises)).flat();
}
}
@@ -267,6 +265,12 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
return formatted;
}
private defaultFilteringFunction(entities: Entity[]): Entity[] {
return entities.filter(
entity => entity.metadata?.annotations?.['backstage.io/techdocs-ref'],
);
}
private static constructDocsIndexUrl(
techDocsBaseUrl: string,
entityInfo: { kind: string; namespace: string; name: string },
plugins/search-backend-module-techdocs/src/module.ts
+43
View File
@@ -25,6 +25,8 @@ import {
createExtensionPoint,
readSchedulerServiceTaskScheduleDefinitionFromConfig,
} from '@backstage/backend-plugin-api';
import { EntityFilterQuery } from '@backstage/catalog-client';
import { Entity } from '@backstage/catalog-model';
import { catalogServiceRef } from '@backstage/plugin-catalog-node/alpha';
import {
DefaultTechDocsCollatorFactory,
@@ -51,6 +53,24 @@ export const techdocsCollatorEntityTransformerExtensionPoint =
id: 'search.techdocsCollator.transformer',
});
/** @public */
export interface TechDocsCollatorEntityFilterExtensionPoint {
setEntityFilterFunction(
filterFunction: (entities: Entity[]) => Entity[],
): void;
setCustomCatalogApiFilters(apiFilters: EntityFilterQuery): void;
}
/**
* Extension point used to filter the entities that the collator will use.
*
* @public
*/
export const techDocsCollatorEntityFilterExtensionPoint =
createExtensionPoint<TechDocsCollatorEntityFilterExtensionPoint>({
id: 'search.techdocsCollator.entityFilter',
});
/**
* @public
* Search backend module for the TechDocs index.
@@ -61,6 +81,8 @@ export default createBackendModule({
register(env) {
let entityTransformer: TechDocsCollatorEntityTransformer | undefined;
let documentTransformer: TechDocsCollatorDocumentTransformer | undefined;
let entityFilterFunction: ((e: Entity[]) => Entity[]) | undefined;
let customCatalogApiFilters: EntityFilterQuery | undefined;
env.registerExtensionPoint(
techdocsCollatorEntityTransformerExtensionPoint,
@@ -84,6 +106,25 @@ export default createBackendModule({
},
);
env.registerExtensionPoint(techDocsCollatorEntityFilterExtensionPoint, {
setEntityFilterFunction(newEntityFilterFunction) {
if (entityFilterFunction) {
throw new Error(
'TechDocs entity filter functions may only be set once',
);
}
entityFilterFunction = newEntityFilterFunction;
},
setCustomCatalogApiFilters(newCatalogApiFilters) {
if (customCatalogApiFilters) {
throw new Error(
'TechDocs catalog entity filters may only be set once',
);
}
customCatalogApiFilters = newCatalogApiFilters;
},
});
env.registerInit({
deps: {
config: coreServices.rootConfig,
@@ -127,6 +168,8 @@ export default createBackendModule({
catalogClient: catalog,
entityTransformer,
documentTransformer,
customCatalogApiFilters,
entityFilterFunction,
}),
});
},