diff --git a/plugins/techdocs-backend/api-report.md b/plugins/techdocs-backend/api-report.md
index 2bf4664f77..155f096614 100644
--- a/plugins/techdocs-backend/api-report.md
+++ b/plugins/techdocs-backend/api-report.md
@@ -3,9 +3,11 @@
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
+///
+
import { CatalogApi } from '@backstage/catalog-client';
import { Config } from '@backstage/config';
-import { DocumentCollator } from '@backstage/search-common';
+import { DocumentCollatorFactory } from '@backstage/search-common';
import { Entity } from '@backstage/catalog-model';
import express from 'express';
import { GeneratorBuilder } from '@backstage/techdocs-common';
@@ -16,14 +18,15 @@ import { PluginCacheManager } from '@backstage/backend-common';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { PreparerBuilder } from '@backstage/techdocs-common';
import { PublisherBase } from '@backstage/techdocs-common';
+import { Readable } from 'stream';
import { TechDocsDocument } from '@backstage/techdocs-common';
import { TokenManager } from '@backstage/backend-common';
// @public
export function createRouter(options: RouterOptions): Promise;
-// @public
-export class DefaultTechDocsCollator implements DocumentCollator {
+// @public @deprecated
+export class DefaultTechDocsCollator {
// (undocumented)
protected applyArgsToFormat(
format: string,
@@ -42,6 +45,21 @@ export class DefaultTechDocsCollator implements DocumentCollator {
readonly visibilityPermission: Permission;
}
+// @public
+export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
+ // (undocumented)
+ static fromConfig(
+ config: Config,
+ options: TechDocsCollatorFactoryOptions,
+ ): DefaultTechDocsCollatorFactory;
+ // (undocumented)
+ getCollator(): Promise;
+ // (undocumented)
+ readonly type: string;
+ // (undocumented)
+ readonly visibilityPermission: Permission;
+}
+
// @public
export interface DocsBuildStrategy {
// (undocumented)
@@ -81,6 +99,17 @@ export type ShouldBuildParameters = {
entity: Entity;
};
+// @public
+export type TechDocsCollatorFactoryOptions = {
+ discovery: PluginEndpointDiscovery;
+ logger: Logger_2;
+ tokenManager: TokenManager;
+ locationTemplate?: string;
+ catalogClient?: CatalogApi;
+ parallelismLimit?: number;
+ legacyPathCasing?: boolean;
+};
+
// @public
export type TechDocsCollatorOptions = {
discovery: PluginEndpointDiscovery;
diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json
index 7e8241bfea..20100f4848 100644
--- a/plugins/techdocs-backend/package.json
+++ b/plugins/techdocs-backend/package.json
@@ -56,6 +56,7 @@
},
"devDependencies": {
"@backstage/cli": "^0.14.1",
+ "@backstage/plugin-search-backend-node": "0.4.7",
"@backstage/test-utils": "^0.2.6",
"@types/dockerode": "^3.3.0",
"msw": "^0.35.0",
diff --git a/plugins/techdocs-backend/src/index.ts b/plugins/techdocs-backend/src/index.ts
index 01acbea2cc..570da95092 100644
--- a/plugins/techdocs-backend/src/index.ts
+++ b/plugins/techdocs-backend/src/index.ts
@@ -29,8 +29,14 @@ export type {
ShouldBuildParameters,
} from './service';
-export { DefaultTechDocsCollator } from './search';
-export type { TechDocsCollatorOptions } from './search';
+export {
+ DefaultTechDocsCollator,
+ DefaultTechDocsCollatorFactory,
+} from './search';
+export type {
+ TechDocsCollatorFactoryOptions,
+ TechDocsCollatorOptions,
+} from './search';
/**
* @deprecated Use directly from @backstage/techdocs-common
diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts
index d9f75767a7..887dc60b1a 100644
--- a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts
+++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts
@@ -72,18 +72,6 @@ const expectedEntities: Entity[] = [
owner: 'someone',
},
},
- {
- apiVersion: 'backstage.io/v1alpha1',
- kind: 'Component',
- metadata: {
- name: 'test-entity',
- description: 'The expected description',
- },
- spec: {
- type: 'some-type',
- lifecycle: 'experimental',
- },
- },
];
describe('DefaultTechDocsCollator with legacyPathCasing configuration', () => {
diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts
index 73c030cc51..d36e1c0ae5 100644
--- a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts
+++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts
@@ -24,7 +24,6 @@ import {
RELATION_OWNED_BY,
stringifyEntityRef,
} from '@backstage/catalog-model';
-import { DocumentCollator } from '@backstage/search-common';
import fetch from 'node-fetch';
import unescape from 'lodash/unescape';
import { Logger } from 'winston';
@@ -69,8 +68,10 @@ type EntityInfo = {
* A search collator responsible for gathering and transforming TechDocs documents.
*
* @public
+ * @deprecated Upgrade to a more recent `@backstage/search-backend-node` and
+ * use `DefaultTechDocsCollatorFactory` instead.
*/
-export class DefaultTechDocsCollator implements DocumentCollator {
+export class DefaultTechDocsCollator {
public readonly type: string = 'techdocs';
public readonly visibilityPermission = catalogEntityReadPermission;
diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts
new file mode 100644
index 0000000000..fafaae71b9
--- /dev/null
+++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts
@@ -0,0 +1,245 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+ getVoidLogger,
+ PluginEndpointDiscovery,
+ TokenManager,
+} from '@backstage/backend-common';
+import { Entity } from '@backstage/catalog-model';
+import { ConfigReader } from '@backstage/config';
+import { TestPipeline } from '@backstage/plugin-search-backend-node';
+import { setupRequestMockHandlers } from '@backstage/test-utils';
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import { Readable } from 'stream';
+import { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory';
+
+const logger = getVoidLogger();
+
+const mockSearchDocIndex = {
+ config: {
+ lang: ['en'],
+ min_search_length: 3,
+ prebuild_index: false,
+ separator: '[\\s\\-]+',
+ },
+ docs: [
+ {
+ location: '',
+ text: 'docs docs docs',
+ title: 'Home',
+ },
+ {
+ location: 'local-development/',
+ text: 'Docs for first subtitle',
+ title: 'Local development',
+ },
+ {
+ location: 'local-development/#development',
+ text: 'Docs for sub-subtitle',
+ title: 'Development',
+ },
+ ],
+};
+
+const expectedEntities: Entity[] = [
+ {
+ apiVersion: 'backstage.io/v1alpha1',
+ kind: 'Component',
+ metadata: {
+ title: 'Test Entity with Docs!',
+ name: 'test-entity-with-docs',
+ description: 'Documented description',
+ annotations: {
+ 'backstage.io/techdocs-ref': './',
+ },
+ },
+ spec: {
+ type: 'dog',
+ lifecycle: 'experimental',
+ owner: 'someone',
+ },
+ },
+];
+
+describe('DefaultTechDocsCollatorFactory', () => {
+ const config = new ConfigReader({});
+ const mockDiscoveryApi: jest.Mocked = {
+ getBaseUrl: jest.fn().mockResolvedValue('http://test-backend'),
+ getExternalBaseUrl: jest.fn(),
+ };
+ const mockTokenManager: jest.Mocked = {
+ getToken: jest.fn().mockResolvedValue({ token: '' }),
+ authenticate: jest.fn(),
+ };
+ const options = {
+ discovery: mockDiscoveryApi,
+ logger: getVoidLogger(),
+ tokenManager: mockTokenManager,
+ };
+
+ it('has expected type', () => {
+ const factory = DefaultTechDocsCollatorFactory.fromConfig(config, options);
+ expect(factory.type).toBe('techdocs');
+ });
+
+ describe('getCollator', () => {
+ let factory: DefaultTechDocsCollatorFactory;
+ let collator: Readable;
+
+ const worker = setupServer();
+ setupRequestMockHandlers(worker);
+
+ beforeEach(async () => {
+ factory = DefaultTechDocsCollatorFactory.fromConfig(config, options);
+ collator = await factory.getCollator();
+
+ worker.use(
+ rest.get(
+ 'http://test-backend/static/docs/default/Component/test-entity-with-docs/search/search_index.json',
+ (_, res, ctx) => res(ctx.status(200), ctx.json(mockSearchDocIndex)),
+ ),
+ rest.get('http://test-backend/entities', (req, res, ctx) => {
+ // Imitate offset/limit pagination.
+ const offset = parseInt(
+ req.url.searchParams.get('offset') || '0',
+ 10,
+ );
+ const limit = parseInt(
+ req.url.searchParams.get('limit') || '500',
+ 10,
+ );
+
+ // Limit 50 corresponds to a case testing pagination.
+ if (limit === 50) {
+ // Return 50 copies of invalid entities on the first request.
+ if (offset === 0) {
+ return res(ctx.status(200), ctx.json(Array(50).fill({})));
+ }
+ // Then just the regular 2 on the second.
+ return res(ctx.status(200), ctx.json(expectedEntities));
+ }
+ return res(
+ ctx.status(200),
+ ctx.json(expectedEntities.slice(offset, limit + offset)),
+ );
+ }),
+ );
+ });
+
+ it('returns a readable stream', async () => {
+ expect(collator).toBeInstanceOf(Readable);
+ });
+
+ it('fetches from the configured catalog and tech docs services', async () => {
+ const pipeline = TestPipeline.withSubject(collator);
+ const { documents } = await pipeline.execute();
+ expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog');
+ expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('techdocs');
+ expect(documents).toHaveLength(mockSearchDocIndex.docs.length);
+ });
+
+ it('should create documents for each tech docs search index', async () => {
+ const pipeline = TestPipeline.withSubject(collator);
+ const { documents } = await pipeline.execute();
+ const entity = expectedEntities[0];
+ documents.forEach((document, idx) => {
+ expect(document).toMatchObject({
+ title: mockSearchDocIndex.docs[idx].title,
+ location: `/docs/default/component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`,
+ text: mockSearchDocIndex.docs[idx].text,
+ namespace: 'default',
+ entityTitle: entity!.metadata.title,
+ componentType: entity!.spec!.type,
+ lifecycle: entity!.spec!.lifecycle,
+ owner: '',
+ kind: entity.kind.toLocaleLowerCase('en-US'),
+ name: entity.metadata.name,
+ });
+ });
+ });
+
+ it('maps a returned entity with a custom locationTemplate', async () => {
+ // Provide an alternate location template.
+ factory = DefaultTechDocsCollatorFactory.fromConfig(config, {
+ discovery: mockDiscoveryApi,
+ tokenManager: mockTokenManager,
+ locationTemplate: '/software/:name',
+ logger,
+ });
+ collator = await factory.getCollator();
+
+ const pipeline = TestPipeline.withSubject(collator);
+ const { documents } = await pipeline.execute();
+
+ expect(documents[0]).toMatchObject({
+ location: '/software/test-entity-with-docs',
+ });
+ });
+
+ it('paginates through catalog entities using batchSize', async () => {
+ // A parallelismLimit of 1 is a catalog limit of 50 per request. Code
+ // above in the /entities handler ensures valid entities are only
+ // returned on the second page.
+ factory = DefaultTechDocsCollatorFactory.fromConfig(config, {
+ ...options,
+ parallelismLimit: 1,
+ });
+ collator = await factory.getCollator();
+
+ const pipeline = TestPipeline.withSubject(collator);
+ const { documents } = await pipeline.execute();
+
+ // Only 1 entity with TechDocs configured multipled by 3 pages.
+ expect(documents).toHaveLength(3);
+ });
+
+ describe('with legacyPathCasing configuration', () => {
+ beforeEach(async () => {
+ const legacyConfig = new ConfigReader({
+ techdocs: {
+ legacyUseCaseSensitiveTripletPaths: true,
+ },
+ });
+ factory = DefaultTechDocsCollatorFactory.fromConfig(
+ legacyConfig,
+ options,
+ );
+ collator = await factory.getCollator();
+ });
+
+ it('should create documents for each tech docs search index', async () => {
+ const pipeline = TestPipeline.withSubject(collator);
+ const { documents } = await pipeline.execute();
+ const entity = expectedEntities[0];
+ documents.forEach((document, idx) => {
+ expect(document).toMatchObject({
+ title: mockSearchDocIndex.docs[idx].title,
+ location: `/docs/default/Component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`,
+ text: mockSearchDocIndex.docs[idx].text,
+ namespace: 'default',
+ entityTitle: entity!.metadata.title,
+ componentType: entity!.spec!.type,
+ lifecycle: entity!.spec!.lifecycle,
+ owner: '',
+ kind: entity.kind,
+ name: entity.metadata.name,
+ });
+ });
+ });
+ });
+ });
+});
diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts
new file mode 100644
index 0000000000..181f032b54
--- /dev/null
+++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts
@@ -0,0 +1,254 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+ PluginEndpointDiscovery,
+ TokenManager,
+} from '@backstage/backend-common';
+import {
+ CatalogApi,
+ CatalogClient,
+ CATALOG_FILTER_EXISTS,
+} from '@backstage/catalog-client';
+import {
+ Entity,
+ parseEntityRef,
+ RELATION_OWNED_BY,
+ stringifyEntityRef,
+} from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common';
+import { DocumentCollatorFactory } from '@backstage/search-common';
+import { TechDocsDocument } from '@backstage/techdocs-common';
+import unescape from 'lodash/unescape';
+import fetch from 'node-fetch';
+import pLimit from 'p-limit';
+import { Readable } from 'stream';
+import { Logger } from 'winston';
+
+interface MkSearchIndexDoc {
+ title: string;
+ text: string;
+ location: string;
+}
+
+/**
+ * Options to configure the TechDocs collator factory
+ *
+ * @public
+ */
+export type TechDocsCollatorFactoryOptions = {
+ discovery: PluginEndpointDiscovery;
+ logger: Logger;
+ tokenManager: TokenManager;
+ locationTemplate?: string;
+ catalogClient?: CatalogApi;
+ parallelismLimit?: number;
+ legacyPathCasing?: boolean;
+};
+
+type EntityInfo = {
+ name: string;
+ namespace: string;
+ kind: string;
+};
+
+/**
+ * A search collator factory responsible for gathering and transforming
+ * TechDocs documents.
+ *
+ * @public
+ */
+export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
+ public readonly type: string = 'techdocs';
+ public readonly visibilityPermission = catalogEntityReadPermission;
+
+ private discovery: PluginEndpointDiscovery;
+ private locationTemplate: string;
+ private readonly logger: Logger;
+ private readonly catalogClient: CatalogApi;
+ private readonly tokenManager: TokenManager;
+ private readonly parallelismLimit: number;
+ private readonly legacyPathCasing: boolean;
+
+ private constructor(options: TechDocsCollatorFactoryOptions) {
+ this.discovery = options.discovery;
+ this.locationTemplate =
+ options.locationTemplate || '/docs/:namespace/:kind/:name/:path';
+ this.logger = options.logger;
+ this.catalogClient =
+ options.catalogClient ||
+ new CatalogClient({ discoveryApi: options.discovery });
+ this.parallelismLimit = options.parallelismLimit ?? 10;
+ this.legacyPathCasing = options.legacyPathCasing ?? false;
+ this.tokenManager = options.tokenManager;
+ }
+
+ static fromConfig(config: Config, options: TechDocsCollatorFactoryOptions) {
+ const legacyPathCasing =
+ config.getOptionalBoolean(
+ 'techdocs.legacyUseCaseSensitiveTripletPaths',
+ ) || false;
+ return new DefaultTechDocsCollatorFactory({ ...options, legacyPathCasing });
+ }
+
+ async getCollator(): Promise {
+ return Readable.from(this.execute());
+ }
+
+ private async *execute(): AsyncGenerator {
+ const limit = pLimit(this.parallelismLimit);
+ const techDocsBaseUrl = await this.discovery.getBaseUrl('techdocs');
+ const { token } = await this.tokenManager.getToken();
+ let entitiesRetrieved = 0;
+ let moreEntitiesToGet = true;
+
+ // Offset/limit pagination is used on the Catalog Client in order to
+ // limit (and allow some control over) memory used by the search backend
+ // at index-time. The batchSize is calculated as a factor of the given
+ // parallelism limit to simplify configuration.
+ const batchSize = this.parallelismLimit * 50;
+ while (moreEntitiesToGet) {
+ const entities = (
+ await this.catalogClient.getEntities(
+ {
+ filter: {
+ 'metadata.annotations.backstage.io/techdocs-ref':
+ CATALOG_FILTER_EXISTS,
+ },
+ fields: [
+ 'kind',
+ 'namespace',
+ 'metadata.annotations',
+ 'metadata.name',
+ 'metadata.title',
+ 'metadata.namespace',
+ 'spec.type',
+ 'spec.lifecycle',
+ 'relations',
+ ],
+ limit: batchSize,
+ offset: entitiesRetrieved,
+ },
+ { token },
+ )
+ ).items;
+
+ // Control looping through entity batches.
+ moreEntitiesToGet = entities.length === batchSize;
+ entitiesRetrieved += entities.length;
+
+ const docPromises = entities
+ .filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref'])
+ .map((entity: Entity) =>
+ limit(async (): Promise => {
+ const entityInfo =
+ DefaultTechDocsCollatorFactory.handleEntityInfoCasing(
+ this.legacyPathCasing,
+ {
+ kind: entity.kind,
+ namespace: entity.metadata.namespace || 'default',
+ name: entity.metadata.name,
+ },
+ );
+
+ try {
+ const searchIndexResponse = await fetch(
+ DefaultTechDocsCollatorFactory.constructDocsIndexUrl(
+ techDocsBaseUrl,
+ entityInfo,
+ ),
+ {
+ headers: {
+ Authorization: `Bearer ${token}`,
+ },
+ },
+ );
+ const searchIndex = await searchIndexResponse.json();
+
+ return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({
+ title: unescape(doc.title),
+ text: unescape(doc.text || ''),
+ location: this.applyArgsToFormat(
+ this.locationTemplate || '/docs/:namespace/:kind/:name/:path',
+ {
+ ...entityInfo,
+ path: doc.location,
+ },
+ ),
+ path: doc.location,
+ ...entityInfo,
+ entityTitle: entity.metadata.title,
+ componentType: entity.spec?.type?.toString() || 'other',
+ lifecycle: (entity.spec?.lifecycle as string) || '',
+ owner: getSimpleEntityOwnerString(entity),
+ authorization: {
+ resourceRef: stringifyEntityRef(entity),
+ },
+ }));
+ } catch (e) {
+ this.logger.debug(
+ `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`,
+ e,
+ );
+ return [];
+ }
+ }),
+ );
+ yield* (await Promise.all(docPromises)).flat();
+ }
+ }
+
+ private applyArgsToFormat(
+ format: string,
+ args: Record,
+ ): string {
+ let formatted = format;
+ for (const [key, value] of Object.entries(args)) {
+ formatted = formatted.replace(`:${key}`, value);
+ }
+ return formatted;
+ }
+
+ private static constructDocsIndexUrl(
+ techDocsBaseUrl: string,
+ entityInfo: { kind: string; namespace: string; name: string },
+ ) {
+ return `${techDocsBaseUrl}/static/docs/${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}/search/search_index.json`;
+ }
+
+ private static handleEntityInfoCasing(
+ legacyPaths: boolean,
+ entityInfo: EntityInfo,
+ ): EntityInfo {
+ return legacyPaths
+ ? entityInfo
+ : Object.entries(entityInfo).reduce((acc, [key, value]) => {
+ return { ...acc, [key]: value.toLocaleLowerCase('en-US') };
+ }, {} as EntityInfo);
+ }
+}
+
+function getSimpleEntityOwnerString(entity: Entity): string {
+ if (entity.relations) {
+ const owner = entity.relations.find(r => r.type === RELATION_OWNED_BY);
+ if (owner) {
+ const { name } = parseEntityRef(owner.targetRef);
+ return name;
+ }
+ }
+ return '';
+}
diff --git a/plugins/techdocs-backend/src/search/index.ts b/plugins/techdocs-backend/src/search/index.ts
index fbbd23b964..68e3b4edc7 100644
--- a/plugins/techdocs-backend/src/search/index.ts
+++ b/plugins/techdocs-backend/src/search/index.ts
@@ -13,6 +13,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-export { DefaultTechDocsCollator } from './DefaultTechDocsCollator';
+export { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory';
+export type { TechDocsCollatorFactoryOptions } from './DefaultTechDocsCollatorFactory';
+
+/**
+ * todo(backstage/techdocs-core): stop exporting these in a future release.
+ */
+export { DefaultTechDocsCollator } from './DefaultTechDocsCollator';
export type { TechDocsCollatorOptions } from './DefaultTechDocsCollator';