From 0087554f5ca9f95de4d1338f4d1099b5077d09e6 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Sat, 26 Feb 2022 20:01:30 +0100 Subject: [PATCH] Update TechDocs Collator to be stream-based Signed-off-by: Eric Peterson --- plugins/techdocs-backend/api-report.md | 35 ++- plugins/techdocs-backend/package.json | 1 + plugins/techdocs-backend/src/index.ts | 10 +- .../search/DefaultTechDocsCollator.test.ts | 12 - .../src/search/DefaultTechDocsCollator.ts | 5 +- .../DefaultTechDocsCollatorFactory.test.ts | 245 +++++++++++++++++ .../search/DefaultTechDocsCollatorFactory.ts | 254 ++++++++++++++++++ plugins/techdocs-backend/src/search/index.ts | 8 +- 8 files changed, 550 insertions(+), 20 deletions(-) create mode 100644 plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts create mode 100644 plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts diff --git a/plugins/techdocs-backend/api-report.md b/plugins/techdocs-backend/api-report.md index 2bf4664f77..155f096614 100644 --- a/plugins/techdocs-backend/api-report.md +++ b/plugins/techdocs-backend/api-report.md @@ -3,9 +3,11 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +/// + import { CatalogApi } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; -import { DocumentCollator } from '@backstage/search-common'; +import { DocumentCollatorFactory } from '@backstage/search-common'; import { Entity } from '@backstage/catalog-model'; import express from 'express'; import { GeneratorBuilder } from '@backstage/techdocs-common'; @@ -16,14 +18,15 @@ import { PluginCacheManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PreparerBuilder } from '@backstage/techdocs-common'; import { PublisherBase } from '@backstage/techdocs-common'; +import { Readable } from 'stream'; import { TechDocsDocument } from '@backstage/techdocs-common'; import { TokenManager } from '@backstage/backend-common'; // @public export function createRouter(options: RouterOptions): Promise; -// @public -export class DefaultTechDocsCollator implements DocumentCollator { +// @public @deprecated +export class DefaultTechDocsCollator { // (undocumented) protected applyArgsToFormat( format: string, @@ -42,6 +45,21 @@ export class DefaultTechDocsCollator implements DocumentCollator { readonly visibilityPermission: Permission; } +// @public +export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { + // (undocumented) + static fromConfig( + config: Config, + options: TechDocsCollatorFactoryOptions, + ): DefaultTechDocsCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type: string; + // (undocumented) + readonly visibilityPermission: Permission; +} + // @public export interface DocsBuildStrategy { // (undocumented) @@ -81,6 +99,17 @@ export type ShouldBuildParameters = { entity: Entity; }; +// @public +export type TechDocsCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + logger: Logger_2; + tokenManager: TokenManager; + locationTemplate?: string; + catalogClient?: CatalogApi; + parallelismLimit?: number; + legacyPathCasing?: boolean; +}; + // @public export type TechDocsCollatorOptions = { discovery: PluginEndpointDiscovery; diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 7e8241bfea..20100f4848 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -56,6 +56,7 @@ }, "devDependencies": { "@backstage/cli": "^0.14.1", + "@backstage/plugin-search-backend-node": "0.4.7", "@backstage/test-utils": "^0.2.6", "@types/dockerode": "^3.3.0", "msw": "^0.35.0", diff --git a/plugins/techdocs-backend/src/index.ts b/plugins/techdocs-backend/src/index.ts index 01acbea2cc..570da95092 100644 --- a/plugins/techdocs-backend/src/index.ts +++ b/plugins/techdocs-backend/src/index.ts @@ -29,8 +29,14 @@ export type { ShouldBuildParameters, } from './service'; -export { DefaultTechDocsCollator } from './search'; -export type { TechDocsCollatorOptions } from './search'; +export { + DefaultTechDocsCollator, + DefaultTechDocsCollatorFactory, +} from './search'; +export type { + TechDocsCollatorFactoryOptions, + TechDocsCollatorOptions, +} from './search'; /** * @deprecated Use directly from @backstage/techdocs-common diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts index d9f75767a7..887dc60b1a 100644 --- a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts @@ -72,18 +72,6 @@ const expectedEntities: Entity[] = [ owner: 'someone', }, }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'test-entity', - description: 'The expected description', - }, - spec: { - type: 'some-type', - lifecycle: 'experimental', - }, - }, ]; describe('DefaultTechDocsCollator with legacyPathCasing configuration', () => { diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts index 73c030cc51..d36e1c0ae5 100644 --- a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts @@ -24,7 +24,6 @@ import { RELATION_OWNED_BY, stringifyEntityRef, } from '@backstage/catalog-model'; -import { DocumentCollator } from '@backstage/search-common'; import fetch from 'node-fetch'; import unescape from 'lodash/unescape'; import { Logger } from 'winston'; @@ -69,8 +68,10 @@ type EntityInfo = { * A search collator responsible for gathering and transforming TechDocs documents. * * @public + * @deprecated Upgrade to a more recent `@backstage/search-backend-node` and + * use `DefaultTechDocsCollatorFactory` instead. */ -export class DefaultTechDocsCollator implements DocumentCollator { +export class DefaultTechDocsCollator { public readonly type: string = 'techdocs'; public readonly visibilityPermission = catalogEntityReadPermission; diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts new file mode 100644 index 0000000000..fafaae71b9 --- /dev/null +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts @@ -0,0 +1,245 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + getVoidLogger, + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { setupRequestMockHandlers } from '@backstage/test-utils'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { Readable } from 'stream'; +import { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory'; + +const logger = getVoidLogger(); + +const mockSearchDocIndex = { + config: { + lang: ['en'], + min_search_length: 3, + prebuild_index: false, + separator: '[\\s\\-]+', + }, + docs: [ + { + location: '', + text: 'docs docs docs', + title: 'Home', + }, + { + location: 'local-development/', + text: 'Docs for first subtitle', + title: 'Local development', + }, + { + location: 'local-development/#development', + text: 'Docs for sub-subtitle', + title: 'Development', + }, + ], +}; + +const expectedEntities: Entity[] = [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + title: 'Test Entity with Docs!', + name: 'test-entity-with-docs', + description: 'Documented description', + annotations: { + 'backstage.io/techdocs-ref': './', + }, + }, + spec: { + type: 'dog', + lifecycle: 'experimental', + owner: 'someone', + }, + }, +]; + +describe('DefaultTechDocsCollatorFactory', () => { + const config = new ConfigReader({}); + const mockDiscoveryApi: jest.Mocked = { + getBaseUrl: jest.fn().mockResolvedValue('http://test-backend'), + getExternalBaseUrl: jest.fn(), + }; + const mockTokenManager: jest.Mocked = { + getToken: jest.fn().mockResolvedValue({ token: '' }), + authenticate: jest.fn(), + }; + const options = { + discovery: mockDiscoveryApi, + logger: getVoidLogger(), + tokenManager: mockTokenManager, + }; + + it('has expected type', () => { + const factory = DefaultTechDocsCollatorFactory.fromConfig(config, options); + expect(factory.type).toBe('techdocs'); + }); + + describe('getCollator', () => { + let factory: DefaultTechDocsCollatorFactory; + let collator: Readable; + + const worker = setupServer(); + setupRequestMockHandlers(worker); + + beforeEach(async () => { + factory = DefaultTechDocsCollatorFactory.fromConfig(config, options); + collator = await factory.getCollator(); + + worker.use( + rest.get( + 'http://test-backend/static/docs/default/Component/test-entity-with-docs/search/search_index.json', + (_, res, ctx) => res(ctx.status(200), ctx.json(mockSearchDocIndex)), + ), + rest.get('http://test-backend/entities', (req, res, ctx) => { + // Imitate offset/limit pagination. + const offset = parseInt( + req.url.searchParams.get('offset') || '0', + 10, + ); + const limit = parseInt( + req.url.searchParams.get('limit') || '500', + 10, + ); + + // Limit 50 corresponds to a case testing pagination. + if (limit === 50) { + // Return 50 copies of invalid entities on the first request. + if (offset === 0) { + return res(ctx.status(200), ctx.json(Array(50).fill({}))); + } + // Then just the regular 2 on the second. + return res(ctx.status(200), ctx.json(expectedEntities)); + } + return res( + ctx.status(200), + ctx.json(expectedEntities.slice(offset, limit + offset)), + ); + }), + ); + }); + + it('returns a readable stream', async () => { + expect(collator).toBeInstanceOf(Readable); + }); + + it('fetches from the configured catalog and tech docs services', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog'); + expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('techdocs'); + expect(documents).toHaveLength(mockSearchDocIndex.docs.length); + }); + + it('should create documents for each tech docs search index', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + const entity = expectedEntities[0]; + documents.forEach((document, idx) => { + expect(document).toMatchObject({ + title: mockSearchDocIndex.docs[idx].title, + location: `/docs/default/component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`, + text: mockSearchDocIndex.docs[idx].text, + namespace: 'default', + entityTitle: entity!.metadata.title, + componentType: entity!.spec!.type, + lifecycle: entity!.spec!.lifecycle, + owner: '', + kind: entity.kind.toLocaleLowerCase('en-US'), + name: entity.metadata.name, + }); + }); + }); + + it('maps a returned entity with a custom locationTemplate', async () => { + // Provide an alternate location template. + factory = DefaultTechDocsCollatorFactory.fromConfig(config, { + discovery: mockDiscoveryApi, + tokenManager: mockTokenManager, + locationTemplate: '/software/:name', + logger, + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + expect(documents[0]).toMatchObject({ + location: '/software/test-entity-with-docs', + }); + }); + + it('paginates through catalog entities using batchSize', async () => { + // A parallelismLimit of 1 is a catalog limit of 50 per request. Code + // above in the /entities handler ensures valid entities are only + // returned on the second page. + factory = DefaultTechDocsCollatorFactory.fromConfig(config, { + ...options, + parallelismLimit: 1, + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + // Only 1 entity with TechDocs configured multipled by 3 pages. + expect(documents).toHaveLength(3); + }); + + describe('with legacyPathCasing configuration', () => { + beforeEach(async () => { + const legacyConfig = new ConfigReader({ + techdocs: { + legacyUseCaseSensitiveTripletPaths: true, + }, + }); + factory = DefaultTechDocsCollatorFactory.fromConfig( + legacyConfig, + options, + ); + collator = await factory.getCollator(); + }); + + it('should create documents for each tech docs search index', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + const entity = expectedEntities[0]; + documents.forEach((document, idx) => { + expect(document).toMatchObject({ + title: mockSearchDocIndex.docs[idx].title, + location: `/docs/default/Component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`, + text: mockSearchDocIndex.docs[idx].text, + namespace: 'default', + entityTitle: entity!.metadata.title, + componentType: entity!.spec!.type, + lifecycle: entity!.spec!.lifecycle, + owner: '', + kind: entity.kind, + name: entity.metadata.name, + }); + }); + }); + }); + }); +}); diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts new file mode 100644 index 0000000000..181f032b54 --- /dev/null +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts @@ -0,0 +1,254 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { + CatalogApi, + CatalogClient, + CATALOG_FILTER_EXISTS, +} from '@backstage/catalog-client'; +import { + Entity, + parseEntityRef, + RELATION_OWNED_BY, + stringifyEntityRef, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common'; +import { DocumentCollatorFactory } from '@backstage/search-common'; +import { TechDocsDocument } from '@backstage/techdocs-common'; +import unescape from 'lodash/unescape'; +import fetch from 'node-fetch'; +import pLimit from 'p-limit'; +import { Readable } from 'stream'; +import { Logger } from 'winston'; + +interface MkSearchIndexDoc { + title: string; + text: string; + location: string; +} + +/** + * Options to configure the TechDocs collator factory + * + * @public + */ +export type TechDocsCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + logger: Logger; + tokenManager: TokenManager; + locationTemplate?: string; + catalogClient?: CatalogApi; + parallelismLimit?: number; + legacyPathCasing?: boolean; +}; + +type EntityInfo = { + name: string; + namespace: string; + kind: string; +}; + +/** + * A search collator factory responsible for gathering and transforming + * TechDocs documents. + * + * @public + */ +export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { + public readonly type: string = 'techdocs'; + public readonly visibilityPermission = catalogEntityReadPermission; + + private discovery: PluginEndpointDiscovery; + private locationTemplate: string; + private readonly logger: Logger; + private readonly catalogClient: CatalogApi; + private readonly tokenManager: TokenManager; + private readonly parallelismLimit: number; + private readonly legacyPathCasing: boolean; + + private constructor(options: TechDocsCollatorFactoryOptions) { + this.discovery = options.discovery; + this.locationTemplate = + options.locationTemplate || '/docs/:namespace/:kind/:name/:path'; + this.logger = options.logger; + this.catalogClient = + options.catalogClient || + new CatalogClient({ discoveryApi: options.discovery }); + this.parallelismLimit = options.parallelismLimit ?? 10; + this.legacyPathCasing = options.legacyPathCasing ?? false; + this.tokenManager = options.tokenManager; + } + + static fromConfig(config: Config, options: TechDocsCollatorFactoryOptions) { + const legacyPathCasing = + config.getOptionalBoolean( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) || false; + return new DefaultTechDocsCollatorFactory({ ...options, legacyPathCasing }); + } + + async getCollator(): Promise { + return Readable.from(this.execute()); + } + + private async *execute(): AsyncGenerator { + const limit = pLimit(this.parallelismLimit); + const techDocsBaseUrl = await this.discovery.getBaseUrl('techdocs'); + const { token } = await this.tokenManager.getToken(); + let entitiesRetrieved = 0; + let moreEntitiesToGet = true; + + // Offset/limit pagination is used on the Catalog Client in order to + // limit (and allow some control over) memory used by the search backend + // at index-time. The batchSize is calculated as a factor of the given + // parallelism limit to simplify configuration. + const batchSize = this.parallelismLimit * 50; + while (moreEntitiesToGet) { + const entities = ( + await this.catalogClient.getEntities( + { + filter: { + 'metadata.annotations.backstage.io/techdocs-ref': + CATALOG_FILTER_EXISTS, + }, + fields: [ + 'kind', + 'namespace', + 'metadata.annotations', + 'metadata.name', + 'metadata.title', + 'metadata.namespace', + 'spec.type', + 'spec.lifecycle', + 'relations', + ], + limit: batchSize, + offset: entitiesRetrieved, + }, + { token }, + ) + ).items; + + // Control looping through entity batches. + moreEntitiesToGet = entities.length === batchSize; + entitiesRetrieved += entities.length; + + const docPromises = entities + .filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref']) + .map((entity: Entity) => + limit(async (): Promise => { + const entityInfo = + DefaultTechDocsCollatorFactory.handleEntityInfoCasing( + this.legacyPathCasing, + { + kind: entity.kind, + namespace: entity.metadata.namespace || 'default', + name: entity.metadata.name, + }, + ); + + try { + const searchIndexResponse = await fetch( + DefaultTechDocsCollatorFactory.constructDocsIndexUrl( + techDocsBaseUrl, + entityInfo, + ), + { + headers: { + Authorization: `Bearer ${token}`, + }, + }, + ); + const searchIndex = await searchIndexResponse.json(); + + return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({ + title: unescape(doc.title), + text: unescape(doc.text || ''), + location: this.applyArgsToFormat( + this.locationTemplate || '/docs/:namespace/:kind/:name/:path', + { + ...entityInfo, + path: doc.location, + }, + ), + path: doc.location, + ...entityInfo, + entityTitle: entity.metadata.title, + componentType: entity.spec?.type?.toString() || 'other', + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: getSimpleEntityOwnerString(entity), + authorization: { + resourceRef: stringifyEntityRef(entity), + }, + })); + } catch (e) { + this.logger.debug( + `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`, + e, + ); + return []; + } + }), + ); + yield* (await Promise.all(docPromises)).flat(); + } + } + + private applyArgsToFormat( + format: string, + args: Record, + ): string { + let formatted = format; + for (const [key, value] of Object.entries(args)) { + formatted = formatted.replace(`:${key}`, value); + } + return formatted; + } + + private static constructDocsIndexUrl( + techDocsBaseUrl: string, + entityInfo: { kind: string; namespace: string; name: string }, + ) { + return `${techDocsBaseUrl}/static/docs/${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}/search/search_index.json`; + } + + private static handleEntityInfoCasing( + legacyPaths: boolean, + entityInfo: EntityInfo, + ): EntityInfo { + return legacyPaths + ? entityInfo + : Object.entries(entityInfo).reduce((acc, [key, value]) => { + return { ...acc, [key]: value.toLocaleLowerCase('en-US') }; + }, {} as EntityInfo); + } +} + +function getSimpleEntityOwnerString(entity: Entity): string { + if (entity.relations) { + const owner = entity.relations.find(r => r.type === RELATION_OWNED_BY); + if (owner) { + const { name } = parseEntityRef(owner.targetRef); + return name; + } + } + return ''; +} diff --git a/plugins/techdocs-backend/src/search/index.ts b/plugins/techdocs-backend/src/search/index.ts index fbbd23b964..68e3b4edc7 100644 --- a/plugins/techdocs-backend/src/search/index.ts +++ b/plugins/techdocs-backend/src/search/index.ts @@ -13,6 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { DefaultTechDocsCollator } from './DefaultTechDocsCollator'; +export { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory'; +export type { TechDocsCollatorFactoryOptions } from './DefaultTechDocsCollatorFactory'; + +/** + * todo(backstage/techdocs-core): stop exporting these in a future release. + */ +export { DefaultTechDocsCollator } from './DefaultTechDocsCollator'; export type { TechDocsCollatorOptions } from './DefaultTechDocsCollator';