Update TechDocs Collator to be stream-based
Signed-off-by: Eric Peterson <ericpeterson@spotify.com>
This commit is contained in:
Eric Peterson
@@ -29,8 +29,14 @@ export type {
|
||||
ShouldBuildParameters,
|
||||
} from './service';
|
||||
|
||||
export { DefaultTechDocsCollator } from './search';
|
||||
export type { TechDocsCollatorOptions } from './search';
|
||||
export {
|
||||
DefaultTechDocsCollator,
|
||||
DefaultTechDocsCollatorFactory,
|
||||
} from './search';
|
||||
export type {
|
||||
TechDocsCollatorFactoryOptions,
|
||||
TechDocsCollatorOptions,
|
||||
} from './search';
|
||||
|
||||
/**
|
||||
* @deprecated Use directly from @backstage/techdocs-common
|
||||
|
||||
@@ -72,18 +72,6 @@ const expectedEntities: Entity[] = [
|
||||
owner: 'someone',
|
||||
},
|
||||
},
|
||||
{
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Component',
|
||||
metadata: {
|
||||
name: 'test-entity',
|
||||
description: 'The expected description',
|
||||
},
|
||||
spec: {
|
||||
type: 'some-type',
|
||||
lifecycle: 'experimental',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe('DefaultTechDocsCollator with legacyPathCasing configuration', () => {
|
||||
|
||||
@@ -24,7 +24,6 @@ import {
|
||||
RELATION_OWNED_BY,
|
||||
stringifyEntityRef,
|
||||
} from '@backstage/catalog-model';
|
||||
import { DocumentCollator } from '@backstage/search-common';
|
||||
import fetch from 'node-fetch';
|
||||
import unescape from 'lodash/unescape';
|
||||
import { Logger } from 'winston';
|
||||
@@ -69,8 +68,10 @@ type EntityInfo = {
|
||||
* A search collator responsible for gathering and transforming TechDocs documents.
|
||||
*
|
||||
* @public
|
||||
* @deprecated Upgrade to a more recent `@backstage/search-backend-node` and
|
||||
* use `DefaultTechDocsCollatorFactory` instead.
|
||||
*/
|
||||
export class DefaultTechDocsCollator implements DocumentCollator {
|
||||
export class DefaultTechDocsCollator {
|
||||
public readonly type: string = 'techdocs';
|
||||
public readonly visibilityPermission = catalogEntityReadPermission;
|
||||
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import {
|
||||
getVoidLogger,
|
||||
PluginEndpointDiscovery,
|
||||
TokenManager,
|
||||
} from '@backstage/backend-common';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { TestPipeline } from '@backstage/plugin-search-backend-node';
|
||||
import { setupRequestMockHandlers } from '@backstage/test-utils';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { Readable } from 'stream';
|
||||
import { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory';
|
||||
|
||||
const logger = getVoidLogger();
|
||||
|
||||
const mockSearchDocIndex = {
|
||||
config: {
|
||||
lang: ['en'],
|
||||
min_search_length: 3,
|
||||
prebuild_index: false,
|
||||
separator: '[\\s\\-]+',
|
||||
},
|
||||
docs: [
|
||||
{
|
||||
location: '',
|
||||
text: 'docs docs docs',
|
||||
title: 'Home',
|
||||
},
|
||||
{
|
||||
location: 'local-development/',
|
||||
text: 'Docs for first subtitle',
|
||||
title: 'Local development',
|
||||
},
|
||||
{
|
||||
location: 'local-development/#development',
|
||||
text: 'Docs for sub-subtitle',
|
||||
title: 'Development',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const expectedEntities: Entity[] = [
|
||||
{
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Component',
|
||||
metadata: {
|
||||
title: 'Test Entity with Docs!',
|
||||
name: 'test-entity-with-docs',
|
||||
description: 'Documented description',
|
||||
annotations: {
|
||||
'backstage.io/techdocs-ref': './',
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
type: 'dog',
|
||||
lifecycle: 'experimental',
|
||||
owner: 'someone',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe('DefaultTechDocsCollatorFactory', () => {
|
||||
const config = new ConfigReader({});
|
||||
const mockDiscoveryApi: jest.Mocked<PluginEndpointDiscovery> = {
|
||||
getBaseUrl: jest.fn().mockResolvedValue('http://test-backend'),
|
||||
getExternalBaseUrl: jest.fn(),
|
||||
};
|
||||
const mockTokenManager: jest.Mocked<TokenManager> = {
|
||||
getToken: jest.fn().mockResolvedValue({ token: '' }),
|
||||
authenticate: jest.fn(),
|
||||
};
|
||||
const options = {
|
||||
discovery: mockDiscoveryApi,
|
||||
logger: getVoidLogger(),
|
||||
tokenManager: mockTokenManager,
|
||||
};
|
||||
|
||||
it('has expected type', () => {
|
||||
const factory = DefaultTechDocsCollatorFactory.fromConfig(config, options);
|
||||
expect(factory.type).toBe('techdocs');
|
||||
});
|
||||
|
||||
describe('getCollator', () => {
|
||||
let factory: DefaultTechDocsCollatorFactory;
|
||||
let collator: Readable;
|
||||
|
||||
const worker = setupServer();
|
||||
setupRequestMockHandlers(worker);
|
||||
|
||||
beforeEach(async () => {
|
||||
factory = DefaultTechDocsCollatorFactory.fromConfig(config, options);
|
||||
collator = await factory.getCollator();
|
||||
|
||||
worker.use(
|
||||
rest.get(
|
||||
'http://test-backend/static/docs/default/Component/test-entity-with-docs/search/search_index.json',
|
||||
(_, res, ctx) => res(ctx.status(200), ctx.json(mockSearchDocIndex)),
|
||||
),
|
||||
rest.get('http://test-backend/entities', (req, res, ctx) => {
|
||||
// Imitate offset/limit pagination.
|
||||
const offset = parseInt(
|
||||
req.url.searchParams.get('offset') || '0',
|
||||
10,
|
||||
);
|
||||
const limit = parseInt(
|
||||
req.url.searchParams.get('limit') || '500',
|
||||
10,
|
||||
);
|
||||
|
||||
// Limit 50 corresponds to a case testing pagination.
|
||||
if (limit === 50) {
|
||||
// Return 50 copies of invalid entities on the first request.
|
||||
if (offset === 0) {
|
||||
return res(ctx.status(200), ctx.json(Array(50).fill({})));
|
||||
}
|
||||
// Then just the regular 2 on the second.
|
||||
return res(ctx.status(200), ctx.json(expectedEntities));
|
||||
}
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.json(expectedEntities.slice(offset, limit + offset)),
|
||||
);
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns a readable stream', async () => {
|
||||
expect(collator).toBeInstanceOf(Readable);
|
||||
});
|
||||
|
||||
it('fetches from the configured catalog and tech docs services', async () => {
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog');
|
||||
expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('techdocs');
|
||||
expect(documents).toHaveLength(mockSearchDocIndex.docs.length);
|
||||
});
|
||||
|
||||
it('should create documents for each tech docs search index', async () => {
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
const entity = expectedEntities[0];
|
||||
documents.forEach((document, idx) => {
|
||||
expect(document).toMatchObject({
|
||||
title: mockSearchDocIndex.docs[idx].title,
|
||||
location: `/docs/default/component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`,
|
||||
text: mockSearchDocIndex.docs[idx].text,
|
||||
namespace: 'default',
|
||||
entityTitle: entity!.metadata.title,
|
||||
componentType: entity!.spec!.type,
|
||||
lifecycle: entity!.spec!.lifecycle,
|
||||
owner: '',
|
||||
kind: entity.kind.toLocaleLowerCase('en-US'),
|
||||
name: entity.metadata.name,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('maps a returned entity with a custom locationTemplate', async () => {
|
||||
// Provide an alternate location template.
|
||||
factory = DefaultTechDocsCollatorFactory.fromConfig(config, {
|
||||
discovery: mockDiscoveryApi,
|
||||
tokenManager: mockTokenManager,
|
||||
locationTemplate: '/software/:name',
|
||||
logger,
|
||||
});
|
||||
collator = await factory.getCollator();
|
||||
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
|
||||
expect(documents[0]).toMatchObject({
|
||||
location: '/software/test-entity-with-docs',
|
||||
});
|
||||
});
|
||||
|
||||
it('paginates through catalog entities using batchSize', async () => {
|
||||
// A parallelismLimit of 1 is a catalog limit of 50 per request. Code
|
||||
// above in the /entities handler ensures valid entities are only
|
||||
// returned on the second page.
|
||||
factory = DefaultTechDocsCollatorFactory.fromConfig(config, {
|
||||
...options,
|
||||
parallelismLimit: 1,
|
||||
});
|
||||
collator = await factory.getCollator();
|
||||
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
|
||||
// Only 1 entity with TechDocs configured multipled by 3 pages.
|
||||
expect(documents).toHaveLength(3);
|
||||
});
|
||||
|
||||
describe('with legacyPathCasing configuration', () => {
|
||||
beforeEach(async () => {
|
||||
const legacyConfig = new ConfigReader({
|
||||
techdocs: {
|
||||
legacyUseCaseSensitiveTripletPaths: true,
|
||||
},
|
||||
});
|
||||
factory = DefaultTechDocsCollatorFactory.fromConfig(
|
||||
legacyConfig,
|
||||
options,
|
||||
);
|
||||
collator = await factory.getCollator();
|
||||
});
|
||||
|
||||
it('should create documents for each tech docs search index', async () => {
|
||||
const pipeline = TestPipeline.withSubject(collator);
|
||||
const { documents } = await pipeline.execute();
|
||||
const entity = expectedEntities[0];
|
||||
documents.forEach((document, idx) => {
|
||||
expect(document).toMatchObject({
|
||||
title: mockSearchDocIndex.docs[idx].title,
|
||||
location: `/docs/default/Component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`,
|
||||
text: mockSearchDocIndex.docs[idx].text,
|
||||
namespace: 'default',
|
||||
entityTitle: entity!.metadata.title,
|
||||
componentType: entity!.spec!.type,
|
||||
lifecycle: entity!.spec!.lifecycle,
|
||||
owner: '',
|
||||
kind: entity.kind,
|
||||
name: entity.metadata.name,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,254 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
PluginEndpointDiscovery,
|
||||
TokenManager,
|
||||
} from '@backstage/backend-common';
|
||||
import {
|
||||
CatalogApi,
|
||||
CatalogClient,
|
||||
CATALOG_FILTER_EXISTS,
|
||||
} from '@backstage/catalog-client';
|
||||
import {
|
||||
Entity,
|
||||
parseEntityRef,
|
||||
RELATION_OWNED_BY,
|
||||
stringifyEntityRef,
|
||||
} from '@backstage/catalog-model';
|
||||
import { Config } from '@backstage/config';
|
||||
import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common';
|
||||
import { DocumentCollatorFactory } from '@backstage/search-common';
|
||||
import { TechDocsDocument } from '@backstage/techdocs-common';
|
||||
import unescape from 'lodash/unescape';
|
||||
import fetch from 'node-fetch';
|
||||
import pLimit from 'p-limit';
|
||||
import { Readable } from 'stream';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
interface MkSearchIndexDoc {
|
||||
title: string;
|
||||
text: string;
|
||||
location: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options to configure the TechDocs collator factory
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type TechDocsCollatorFactoryOptions = {
|
||||
discovery: PluginEndpointDiscovery;
|
||||
logger: Logger;
|
||||
tokenManager: TokenManager;
|
||||
locationTemplate?: string;
|
||||
catalogClient?: CatalogApi;
|
||||
parallelismLimit?: number;
|
||||
legacyPathCasing?: boolean;
|
||||
};
|
||||
|
||||
type EntityInfo = {
|
||||
name: string;
|
||||
namespace: string;
|
||||
kind: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* A search collator factory responsible for gathering and transforming
|
||||
* TechDocs documents.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
|
||||
public readonly type: string = 'techdocs';
|
||||
public readonly visibilityPermission = catalogEntityReadPermission;
|
||||
|
||||
private discovery: PluginEndpointDiscovery;
|
||||
private locationTemplate: string;
|
||||
private readonly logger: Logger;
|
||||
private readonly catalogClient: CatalogApi;
|
||||
private readonly tokenManager: TokenManager;
|
||||
private readonly parallelismLimit: number;
|
||||
private readonly legacyPathCasing: boolean;
|
||||
|
||||
private constructor(options: TechDocsCollatorFactoryOptions) {
|
||||
this.discovery = options.discovery;
|
||||
this.locationTemplate =
|
||||
options.locationTemplate || '/docs/:namespace/:kind/:name/:path';
|
||||
this.logger = options.logger;
|
||||
this.catalogClient =
|
||||
options.catalogClient ||
|
||||
new CatalogClient({ discoveryApi: options.discovery });
|
||||
this.parallelismLimit = options.parallelismLimit ?? 10;
|
||||
this.legacyPathCasing = options.legacyPathCasing ?? false;
|
||||
this.tokenManager = options.tokenManager;
|
||||
}
|
||||
|
||||
static fromConfig(config: Config, options: TechDocsCollatorFactoryOptions) {
|
||||
const legacyPathCasing =
|
||||
config.getOptionalBoolean(
|
||||
'techdocs.legacyUseCaseSensitiveTripletPaths',
|
||||
) || false;
|
||||
return new DefaultTechDocsCollatorFactory({ ...options, legacyPathCasing });
|
||||
}
|
||||
|
||||
async getCollator(): Promise<Readable> {
|
||||
return Readable.from(this.execute());
|
||||
}
|
||||
|
||||
private async *execute(): AsyncGenerator<TechDocsDocument, void, undefined> {
|
||||
const limit = pLimit(this.parallelismLimit);
|
||||
const techDocsBaseUrl = await this.discovery.getBaseUrl('techdocs');
|
||||
const { token } = await this.tokenManager.getToken();
|
||||
let entitiesRetrieved = 0;
|
||||
let moreEntitiesToGet = true;
|
||||
|
||||
// Offset/limit pagination is used on the Catalog Client in order to
|
||||
// limit (and allow some control over) memory used by the search backend
|
||||
// at index-time. The batchSize is calculated as a factor of the given
|
||||
// parallelism limit to simplify configuration.
|
||||
const batchSize = this.parallelismLimit * 50;
|
||||
while (moreEntitiesToGet) {
|
||||
const entities = (
|
||||
await this.catalogClient.getEntities(
|
||||
{
|
||||
filter: {
|
||||
'metadata.annotations.backstage.io/techdocs-ref':
|
||||
CATALOG_FILTER_EXISTS,
|
||||
},
|
||||
fields: [
|
||||
'kind',
|
||||
'namespace',
|
||||
'metadata.annotations',
|
||||
'metadata.name',
|
||||
'metadata.title',
|
||||
'metadata.namespace',
|
||||
'spec.type',
|
||||
'spec.lifecycle',
|
||||
'relations',
|
||||
],
|
||||
limit: batchSize,
|
||||
offset: entitiesRetrieved,
|
||||
},
|
||||
{ token },
|
||||
)
|
||||
).items;
|
||||
|
||||
// Control looping through entity batches.
|
||||
moreEntitiesToGet = entities.length === batchSize;
|
||||
entitiesRetrieved += entities.length;
|
||||
|
||||
const docPromises = entities
|
||||
.filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref'])
|
||||
.map((entity: Entity) =>
|
||||
limit(async (): Promise<TechDocsDocument[]> => {
|
||||
const entityInfo =
|
||||
DefaultTechDocsCollatorFactory.handleEntityInfoCasing(
|
||||
this.legacyPathCasing,
|
||||
{
|
||||
kind: entity.kind,
|
||||
namespace: entity.metadata.namespace || 'default',
|
||||
name: entity.metadata.name,
|
||||
},
|
||||
);
|
||||
|
||||
try {
|
||||
const searchIndexResponse = await fetch(
|
||||
DefaultTechDocsCollatorFactory.constructDocsIndexUrl(
|
||||
techDocsBaseUrl,
|
||||
entityInfo,
|
||||
),
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
const searchIndex = await searchIndexResponse.json();
|
||||
|
||||
return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({
|
||||
title: unescape(doc.title),
|
||||
text: unescape(doc.text || ''),
|
||||
location: this.applyArgsToFormat(
|
||||
this.locationTemplate || '/docs/:namespace/:kind/:name/:path',
|
||||
{
|
||||
...entityInfo,
|
||||
path: doc.location,
|
||||
},
|
||||
),
|
||||
path: doc.location,
|
||||
...entityInfo,
|
||||
entityTitle: entity.metadata.title,
|
||||
componentType: entity.spec?.type?.toString() || 'other',
|
||||
lifecycle: (entity.spec?.lifecycle as string) || '',
|
||||
owner: getSimpleEntityOwnerString(entity),
|
||||
authorization: {
|
||||
resourceRef: stringifyEntityRef(entity),
|
||||
},
|
||||
}));
|
||||
} catch (e) {
|
||||
this.logger.debug(
|
||||
`Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`,
|
||||
e,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}),
|
||||
);
|
||||
yield* (await Promise.all(docPromises)).flat();
|
||||
}
|
||||
}
|
||||
|
||||
private applyArgsToFormat(
|
||||
format: string,
|
||||
args: Record<string, string>,
|
||||
): string {
|
||||
let formatted = format;
|
||||
for (const [key, value] of Object.entries(args)) {
|
||||
formatted = formatted.replace(`:${key}`, value);
|
||||
}
|
||||
return formatted;
|
||||
}
|
||||
|
||||
private static constructDocsIndexUrl(
|
||||
techDocsBaseUrl: string,
|
||||
entityInfo: { kind: string; namespace: string; name: string },
|
||||
) {
|
||||
return `${techDocsBaseUrl}/static/docs/${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}/search/search_index.json`;
|
||||
}
|
||||
|
||||
private static handleEntityInfoCasing(
|
||||
legacyPaths: boolean,
|
||||
entityInfo: EntityInfo,
|
||||
): EntityInfo {
|
||||
return legacyPaths
|
||||
? entityInfo
|
||||
: Object.entries(entityInfo).reduce((acc, [key, value]) => {
|
||||
return { ...acc, [key]: value.toLocaleLowerCase('en-US') };
|
||||
}, {} as EntityInfo);
|
||||
}
|
||||
}
|
||||
|
||||
function getSimpleEntityOwnerString(entity: Entity): string {
|
||||
if (entity.relations) {
|
||||
const owner = entity.relations.find(r => r.type === RELATION_OWNED_BY);
|
||||
if (owner) {
|
||||
const { name } = parseEntityRef(owner.targetRef);
|
||||
return name;
|
||||
}
|
||||
}
|
||||
return '';
|
||||
}
|
||||
@@ -13,6 +13,12 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export { DefaultTechDocsCollator } from './DefaultTechDocsCollator';
|
||||
|
||||
export { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory';
|
||||
export type { TechDocsCollatorFactoryOptions } from './DefaultTechDocsCollatorFactory';
|
||||
|
||||
/**
|
||||
* todo(backstage/techdocs-core): stop exporting these in a future release.
|
||||
*/
|
||||
export { DefaultTechDocsCollator } from './DefaultTechDocsCollator';
|
||||
export type { TechDocsCollatorOptions } from './DefaultTechDocsCollator';
|
||||
|
||||
Reference in New Issue
Block a user