paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 36 Packages Projects Releases Wiki Activity

Merge pull request #25497 from alexef/techdocs-tags

Browse Source 
Techdocs: allow custom collation of mkdocs search document
This commit is contained in:
Alex Lorenzi
2024-10-09 13:32:56 -04:00
committed by GitHub
parent 6cdbca351c 1c9b5545d2
commit 1de80f20f6
12 changed files with 217 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/funny-dancers-drum.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-search-backend-module-techdocs': minor
---
Refactor TechDocs collator, enable clients to override the mkdocs search index transformer, so that per document properties (like tags) can be added to Backstage search index.
docs/features/search/how-to-guides.md
+11 -1
View File
@@ -86,17 +86,27 @@ const techDocsEntityTransformer: TechDocsCollatorEntityTransformer = (
) => {
return {
// add more fields to the index
...defaultTechDocsCollatorEntityTransformer(entity),
tags: entity.metadata.tags,
};
};
const techDocsDocumentTransformer: TechDocsCollatorDocumentTransformer = (
doc: MkSearchIndexDoc,
) => {
return {
// add more fields to the index
bost: doc.boost,
};
};
indexBuilder.addCollator({
collator: DefaultTechDocsCollatorFactory.fromConfig(env.config, {
discovery: env.discovery,
tokenManager: env.tokenManager,
/* highlight-add-next-line */
entityTransformer: techDocsEntityTransformer,
/* highlight-add-next-line */
documentTransformer: techDocsDocumentTransformer,
}),
});
```
plugins/search-backend-module-techdocs/report-alpha.api.md
+6
View File
@@ -5,6 +5,7 @@
```ts
import { BackendFeature } from '@backstage/backend-plugin-api';
import { ExtensionPoint } from '@backstage/backend-plugin-api';
import { TechDocsCollatorDocumentTransformer } from '@backstage/plugin-search-backend-module-techdocs';
import { TechDocsCollatorEntityTransformer } from '@backstage/plugin-search-backend-module-techdocs';
// @alpha
@@ -13,6 +14,10 @@ export default _default;
// @alpha (undocumented)
export interface TechDocsCollatorEntityTransformerExtensionPoint {
// (undocumented)
setDocumentTransformer(
transformer: TechDocsCollatorDocumentTransformer,
): void;
// (undocumented)
setTransformer(transformer: TechDocsCollatorEntityTransformer): void;
}
@@ -24,6 +29,7 @@ export const techdocsCollatorEntityTransformerExtensionPoint: ExtensionPoint<Tec
//
// src/alpha.d.ts:3:1 - (ae-undocumented) Missing documentation for "TechDocsCollatorEntityTransformerExtensionPoint".
// src/alpha.d.ts:4:5 - (ae-undocumented) Missing documentation for "setTransformer".
// src/alpha.d.ts:5:5 - (ae-undocumented) Missing documentation for "setDocumentTransformer".
// (No @packageDocumentation comment for this package)
```
plugins/search-backend-module-techdocs/report.api.md
+40 -5
View File
@@ -36,10 +36,38 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
readonly visibilityPermission: Permission;
}
// @public (undocumented)
export interface MkSearchIndexDoc {
// (undocumented)
location: string;
// (undocumented)
tags?: string[];
// (undocumented)
text: string;
// (undocumented)
title: string;
}
// @public (undocumented)
export type TechDocsCollatorDocumentTransformer = (
doc: MkSearchIndexDoc,
) => Partial<
Omit<
TechDocsDocument,
| 'location'
| 'authorization'
| 'kind'
| 'namespace'
| 'name'
| 'lifecycle'
| 'owner'
>
>;
// @public (undocumented)
export type TechDocsCollatorEntityTransformer = (
entity: Entity,
) => Omit<TechDocsDocument, 'location' | 'authorization'>;
) => Partial<Omit<TechDocsDocument, 'location' | 'authorization'>>;
// @public @deprecated
export type TechDocsCollatorFactoryOptions = {
@@ -53,14 +81,21 @@ export type TechDocsCollatorFactoryOptions = {
parallelismLimit?: number;
legacyPathCasing?: boolean;
entityTransformer?: TechDocsCollatorEntityTransformer;
documentTransformer?: TechDocsCollatorDocumentTransformer;
};
// Warnings were encountered during analysis:
//
// src/collators/DefaultTechDocsCollatorFactory.d.ts:36:5 - (ae-undocumented) Missing documentation for "type".
// src/collators/DefaultTechDocsCollatorFactory.d.ts:37:5 - (ae-undocumented) Missing documentation for "visibilityPermission".
// src/collators/DefaultTechDocsCollatorFactory.d.ts:47:5 - (ae-undocumented) Missing documentation for "fromConfig".
// src/collators/DefaultTechDocsCollatorFactory.d.ts:48:5 - (ae-undocumented) Missing documentation for "getCollator".
// src/collators/DefaultTechDocsCollatorFactory.d.ts:38:5 - (ae-undocumented) Missing documentation for "type".
// src/collators/DefaultTechDocsCollatorFactory.d.ts:39:5 - (ae-undocumented) Missing documentation for "visibilityPermission".
// src/collators/DefaultTechDocsCollatorFactory.d.ts:50:5 - (ae-undocumented) Missing documentation for "fromConfig".
// src/collators/DefaultTechDocsCollatorFactory.d.ts:51:5 - (ae-undocumented) Missing documentation for "getCollator".
// src/collators/TechDocsCollatorDocumentTransformer.d.ts:3:1 - (ae-undocumented) Missing documentation for "MkSearchIndexDoc".
// src/collators/TechDocsCollatorDocumentTransformer.d.ts:4:5 - (ae-undocumented) Missing documentation for "title".
// src/collators/TechDocsCollatorDocumentTransformer.d.ts:5:5 - (ae-undocumented) Missing documentation for "text".
// src/collators/TechDocsCollatorDocumentTransformer.d.ts:6:5 - (ae-undocumented) Missing documentation for "location".
// src/collators/TechDocsCollatorDocumentTransformer.d.ts:7:5 - (ae-undocumented) Missing documentation for "tags".
// src/collators/TechDocsCollatorDocumentTransformer.d.ts:10:1 - (ae-undocumented) Missing documentation for "TechDocsCollatorDocumentTransformer".
// src/collators/TechDocsCollatorEntityTransformer.d.ts:4:1 - (ae-undocumented) Missing documentation for "TechDocsCollatorEntityTransformer".
// src/collators/defaultTechDocsCollatorEntityTransformer.d.ts:3:22 - (ae-undocumented) Missing documentation for "defaultTechDocsCollatorEntityTransformer".
```
plugins/search-backend-module-techdocs/src/alpha.ts
+18 -4
View File
@@ -28,6 +28,7 @@ import {
import { catalogServiceRef } from '@backstage/plugin-catalog-node/alpha';
import {
DefaultTechDocsCollatorFactory,
TechDocsCollatorDocumentTransformer,
TechDocsCollatorEntityTransformer,
} from '@backstage/plugin-search-backend-module-techdocs';
import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha';
@@ -35,6 +36,9 @@ import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-back
/** @alpha */
export interface TechDocsCollatorEntityTransformerExtensionPoint {
setTransformer(transformer: TechDocsCollatorEntityTransformer): void;
setDocumentTransformer(
transformer: TechDocsCollatorDocumentTransformer,
): void;
}
/**
@@ -55,18 +59,27 @@ export default createBackendModule({
pluginId: 'search',
moduleId: 'techdocs-collator',
register(env) {
let transformer: TechDocsCollatorEntityTransformer | undefined;
let entityTransformer: TechDocsCollatorEntityTransformer | undefined;
let documentTransformer: TechDocsCollatorDocumentTransformer | undefined;
env.registerExtensionPoint(
techdocsCollatorEntityTransformerExtensionPoint,
{
setTransformer(newTransformer) {
if (transformer) {
if (entityTransformer) {
throw new Error(
'TechDocs collator entity transformer may only be set once',
);
}
transformer = newTransformer;
entityTransformer = newTransformer;
},
setDocumentTransformer(newTransformer) {
if (documentTransformer) {
throw new Error(
'TechDocs collator document transformer may only be set once',
);
}
documentTransformer = newTransformer;
},
},
);
@@ -112,7 +125,8 @@ export default createBackendModule({
httpAuth,
logger,
catalogClient: catalog,
entityTransformer: transformer,
entityTransformer,
documentTransformer,
}),
});
},
plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.test.ts
+42 -2
View File
@@ -24,9 +24,12 @@ import { rest } from 'msw';
import { setupServer } from 'msw/node';
import { Readable } from 'stream';
import { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory';
import { defaultTechDocsCollatorEntityTransformer } from './defaultTechDocsCollatorEntityTransformer';
import { TechDocsCollatorEntityTransformer } from './TechDocsCollatorEntityTransformer';
import { DiscoveryService } from '@backstage/backend-plugin-api';
import {
MkSearchIndexDoc,
TechDocsCollatorDocumentTransformer,
} from './TechDocsCollatorDocumentTransformer';
const logger = mockServices.logger.mock();
@@ -254,11 +257,11 @@ describe('DefaultTechDocsCollatorFactory', () => {
});
it('should transform the entity using the entityTransformer function', async () => {
// @ts-ignore
const entityTransformer: TechDocsCollatorEntityTransformer = (
entity: Entity,
) => {
return {
...defaultTechDocsCollatorEntityTransformer(entity),
tags: entity.metadata.tags,
};
};
@@ -289,5 +292,42 @@ describe('DefaultTechDocsCollatorFactory', () => {
});
});
});
it('should transform the doc using the documentTransformer function', async () => {
// @ts-ignore
const documentTransformer: TechDocsCollatorDocumentTransformer = (
_: MkSearchIndexDoc,
) => {
return {
tags: ['static-tag'],
};
};
factory = DefaultTechDocsCollatorFactory.fromConfig(config, {
...options,
documentTransformer,
});
collator = await factory.getCollator();
const pipeline = TestPipeline.fromCollator(collator);
const { documents } = await pipeline.execute();
const entity = expectedEntities[0];
documents.forEach((document, idx) => {
expect(document).toMatchObject({
title: mockSearchDocIndex.docs[idx].title,
location: `/docs/default/component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`,
text: mockSearchDocIndex.docs[idx].text,
namespace: 'default',
entityTitle: entity!.metadata.title,
componentType: entity!.spec!.type,
lifecycle: entity!.spec!.lifecycle,
owner: '',
kind: entity.kind.toLocaleLowerCase('en-US'),
name: entity.metadata.name,
tags: ['static-tag'],
});
});
});
});
});
plugins/search-backend-module-techdocs/src/collators/DefaultTechDocsCollatorFactory.ts
+12 -12
View File
@@ -34,12 +34,16 @@ import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common/al
import { Permission } from '@backstage/plugin-permission-common';
import { DocumentCollatorFactory } from '@backstage/plugin-search-common';
import { TechDocsDocument } from '@backstage/plugin-techdocs-node';
import unescape from 'lodash/unescape';
import fetch from 'node-fetch';
import pLimit from 'p-limit';
import { Readable } from 'stream';
import { TechDocsCollatorEntityTransformer } from './TechDocsCollatorEntityTransformer';
import {
MkSearchIndexDoc,
TechDocsCollatorDocumentTransformer,
} from './TechDocsCollatorDocumentTransformer';
import { defaultTechDocsCollatorEntityTransformer } from './defaultTechDocsCollatorEntityTransformer';
import { defaultTechDocsCollatorDocumentTransformer } from './defaultTechDocsCollatorDocumentTransformer';
import {
AuthService,
DiscoveryService,
@@ -47,12 +51,6 @@ import {
LoggerService,
} from '@backstage/backend-plugin-api';
interface MkSearchIndexDoc {
title: string;
text: string;
location: string;
}
/**
* Options to configure the TechDocs collator factory
*
@@ -70,6 +68,7 @@ export type TechDocsCollatorFactoryOptions = {
parallelismLimit?: number;
legacyPathCasing?: boolean;
entityTransformer?: TechDocsCollatorEntityTransformer;
documentTransformer?: TechDocsCollatorDocumentTransformer;
};
type EntityInfo = {
@@ -98,6 +97,7 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
private readonly parallelismLimit: number;
private readonly legacyPathCasing: boolean;
private entityTransformer: TechDocsCollatorEntityTransformer;
private documentTransformer: TechDocsCollatorDocumentTransformer;
private constructor(options: TechDocsCollatorFactoryOptions) {
this.discovery = options.discovery;
@@ -109,8 +109,8 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
new CatalogClient({ discoveryApi: options.discovery });
this.parallelismLimit = options.parallelismLimit ?? 10;
this.legacyPathCasing = options.legacyPathCasing ?? false;
this.entityTransformer =
options.entityTransformer ?? defaultTechDocsCollatorEntityTransformer;
this.entityTransformer = options.entityTransformer ?? (() => ({}));
this.documentTransformer = options.documentTransformer ?? (() => ({}));
this.auth = createLegacyAuthAdapters({
auth: options.auth,
@@ -224,9 +224,10 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
]);
return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({
...defaultTechDocsCollatorEntityTransformer(entity),
...defaultTechDocsCollatorDocumentTransformer(doc),
...this.entityTransformer(entity),
title: unescape(doc.title),
text: unescape(doc.text || ''),
...this.documentTransformer(doc),
location: this.applyArgsToFormat(
this.locationTemplate || '/docs/:namespace/:kind/:name/:path',
{
@@ -234,7 +235,6 @@ export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
path: doc.location,
},
),
path: doc.location,
...entityInfo,
entityTitle: entity.metadata.title,
componentType: entity.spec?.type?.toString() || 'other',
plugins/search-backend-module-techdocs/src/collators/TechDocsCollatorDocumentTransformer.ts
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TechDocsDocument } from '@backstage/plugin-techdocs-node';
/** @public */
export interface MkSearchIndexDoc {
title: string;
text: string;
location: string;
tags?: string[];
}
/** @public */
export type TechDocsCollatorDocumentTransformer = (
doc: MkSearchIndexDoc,
) => Partial<
Omit<
TechDocsDocument,
| 'location'
| 'authorization'
| 'kind'
| 'namespace'
| 'name'
| 'lifecycle'
| 'owner'
>
>;
plugins/search-backend-module-techdocs/src/collators/TechDocsCollatorEntityTransformer.ts
+1 -1
View File
@@ -20,4 +20,4 @@ import { TechDocsDocument } from '@backstage/plugin-techdocs-node';
/** @public */
export type TechDocsCollatorEntityTransformer = (
entity: Entity,
) => Omit<TechDocsDocument, 'location' | 'authorization'>;
) => Partial<Omit<TechDocsDocument, 'location' | 'authorization'>>;
plugins/search-backend-module-techdocs/src/collators/defaultTechDocsCollatorDocumentTransformer.ts
+30
View File
@@ -0,0 +1,30 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import unescape from 'lodash/unescape';
import {
TechDocsCollatorDocumentTransformer,
MkSearchIndexDoc,
} from './TechDocsCollatorDocumentTransformer';
/** @public */
export const defaultTechDocsCollatorDocumentTransformer: TechDocsCollatorDocumentTransformer =
(doc: MkSearchIndexDoc) => {
return {
title: unescape(doc.title),
text: unescape(doc.text || ''),
path: doc.location,
};
};
plugins/search-backend-module-techdocs/src/collators/index.ts
+5
View File
@@ -21,3 +21,8 @@ export type { TechDocsCollatorFactoryOptions } from './DefaultTechDocsCollatorFa
export { defaultTechDocsCollatorEntityTransformer } from './defaultTechDocsCollatorEntityTransformer';
export type { TechDocsCollatorEntityTransformer } from './TechDocsCollatorEntityTransformer';
export type {
TechDocsCollatorDocumentTransformer,
MkSearchIndexDoc,
} from './TechDocsCollatorDocumentTransformer';
storybook/yarn.lock
+7 -72
View File
@@ -4316,17 +4316,7 @@ __metadata:
languageName: node
linkType: hard
"call-bind@npm:^1.0.0, call-bind@npm:^1.0.2":
version: 1.0.2
resolution: "call-bind@npm:1.0.2"
dependencies:
function-bind: ^1.1.1
get-intrinsic: ^1.0.2
checksum: f8e31de9d19988a4b80f3e704788c4a2d6b6f3d17cfec4f57dc29ced450c53a49270dc66bf0fbd693329ee948dd33e6c90a329519aef17474a4d961e8d6426b0
languageName: node
linkType: hard
"call-bind@npm:^1.0.7":
"call-bind@npm:^1.0.2, call-bind@npm:^1.0.7":
version: 1.0.7
resolution: "call-bind@npm:1.0.7"
dependencies:
@@ -6171,14 +6161,7 @@ __metadata:
languageName: node
linkType: hard
"function-bind@npm:^1.1.1":
version: 1.1.1
resolution: "function-bind@npm:1.1.1"
checksum: b32fbaebb3f8ec4969f033073b43f5c8befbb58f1a79e12f1d7490358150359ebd92f49e72ff0144f65f2c48ea2a605bff2d07965f548f6474fd8efd95bf361a
languageName: node
linkType: hard
"function-bind@npm:^1.1.2":
"function-bind@npm:^1.1.1, function-bind@npm:^1.1.2":
version: 1.1.2
resolution: "function-bind@npm:1.1.2"
checksum: 2b0ff4ce708d99715ad14a6d1f894e2a83242e4a52ccfcefaee5e40050562e5f6dafc1adbb4ce2d4ab47279a45dc736ab91ea5042d843c3c092820dfe032efb1
@@ -6251,19 +6234,7 @@ __metadata:
languageName: node
linkType: hard
"get-intrinsic@npm:^1.0.2, get-intrinsic@npm:^1.1.1, get-intrinsic@npm:^1.1.3, get-intrinsic@npm:^1.2.0":
version: 1.2.1
resolution: "get-intrinsic@npm:1.2.1"
dependencies:
function-bind: ^1.1.1
has: ^1.0.3
has-proto: ^1.0.1
has-symbols: ^1.0.3
checksum: 5b61d88552c24b0cf6fa2d1b3bc5459d7306f699de060d76442cce49a4721f52b8c560a33ab392cf5575b7810277d54ded9d4d39a1ea61855619ebc005aa7e5f
languageName: node
linkType: hard
"get-intrinsic@npm:^1.2.4":
"get-intrinsic@npm:^1.1.1, get-intrinsic@npm:^1.1.3, get-intrinsic@npm:^1.2.0, get-intrinsic@npm:^1.2.4":
version: 1.2.4
resolution: "get-intrinsic@npm:1.2.4"
dependencies:
@@ -6498,16 +6469,7 @@ __metadata:
languageName: node
linkType: hard
"has-property-descriptors@npm:^1.0.0":
version: 1.0.0
resolution: "has-property-descriptors@npm:1.0.0"
dependencies:
get-intrinsic: ^1.1.1
checksum: a6d3f0a266d0294d972e354782e872e2fe1b6495b321e6ef678c9b7a06a40408a6891817350c62e752adced73a94ac903c54734fee05bf65b1905ee1368194bb
languageName: node
linkType: hard
"has-property-descriptors@npm:^1.0.2":
"has-property-descriptors@npm:^1.0.0, has-property-descriptors@npm:^1.0.2":
version: 1.0.2
resolution: "has-property-descriptors@npm:1.0.2"
dependencies:
@@ -8645,14 +8607,7 @@ __metadata:
languageName: node
linkType: hard
"object-inspect@npm:^1.12.0, object-inspect@npm:^1.9.0":
version: 1.12.2
resolution: "object-inspect@npm:1.12.2"
checksum: a534fc1b8534284ed71f25ce3a496013b7ea030f3d1b77118f6b7b1713829262be9e6243acbcb3ef8c626e2b64186112cb7f6db74e37b2789b9c789ca23048b2
languageName: node
linkType: hard
"object-inspect@npm:^1.13.1":
"object-inspect@npm:^1.12.0, object-inspect@npm:^1.13.1":
version: 1.13.2
resolution: "object-inspect@npm:1.13.2"
checksum: 9f850b3c045db60e0e97746e809ee4090d6ce62195af17dd1e9438ac761394a7d8ec4f7906559aea5424eaf61e35d3e53feded2ccd5f62fcc7d9670d3c8eb353
@@ -9581,7 +9536,7 @@ __metadata:
languageName: node
linkType: hard
"qs@npm:6.13.0":
"qs@npm:6.13.0, qs@npm:^6.10.0":
version: 6.13.0
resolution: "qs@npm:6.13.0"
dependencies:
@@ -9590,15 +9545,6 @@ __metadata:
languageName: node
linkType: hard
"qs@npm:^6.10.0":
version: 6.11.0
resolution: "qs@npm:6.11.0"
dependencies:
side-channel: ^1.0.4
checksum: 6e1f29dd5385f7488ec74ac7b6c92f4d09a90408882d0c208414a34dd33badc1a621019d4c799a3df15ab9b1d0292f97c1dd71dc7c045e69f81a8064e5af7297
languageName: node
linkType: hard
"queue-microtask@npm:^1.2.2":
version: 1.2.3
resolution: "queue-microtask@npm:1.2.3"
@@ -10477,18 +10423,7 @@ __metadata:
languageName: node
linkType: hard
"side-channel@npm:^1.0.4":
version: 1.0.4
resolution: "side-channel@npm:1.0.4"
dependencies:
call-bind: ^1.0.0
get-intrinsic: ^1.0.2
object-inspect: ^1.9.0
checksum: 351e41b947079c10bd0858364f32bb3a7379514c399edb64ab3dce683933483fc63fb5e4efe0a15a2e8a7e3c436b6a91736ddb8d8c6591b0460a24bb4a1ee245
languageName: node
linkType: hard
"side-channel@npm:^1.0.6":
"side-channel@npm:^1.0.4, side-channel@npm:^1.0.6":
version: 1.0.6
resolution: "side-channel@npm:1.0.6"
dependencies: