paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 43 Packages Projects Releases Wiki Activity

Update TechDocs Collator to be stream-based

Browse Source 
Signed-off-by: Eric Peterson <ericpeterson@spotify.com>
This commit is contained in:
Eric Peterson
2022-02-26 20:01:30 +01:00
parent 45f68efd40
commit 0087554f5c
8 changed files with 550 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/techdocs-backend/api-report.md
+32 -3
View File
@@ -3,9 +3,11 @@
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
/// <reference types="node" />
import { CatalogApi } from '@backstage/catalog-client';
import { Config } from '@backstage/config';
import { DocumentCollator } from '@backstage/search-common';
import { DocumentCollatorFactory } from '@backstage/search-common';
import { Entity } from '@backstage/catalog-model';
import express from 'express';
import { GeneratorBuilder } from '@backstage/techdocs-common';
@@ -16,14 +18,15 @@ import { PluginCacheManager } from '@backstage/backend-common';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { PreparerBuilder } from '@backstage/techdocs-common';
import { PublisherBase } from '@backstage/techdocs-common';
import { Readable } from 'stream';
import { TechDocsDocument } from '@backstage/techdocs-common';
import { TokenManager } from '@backstage/backend-common';
// @public
export function createRouter(options: RouterOptions): Promise<express.Router>;
// @public
export class DefaultTechDocsCollator implements DocumentCollator {
// @public @deprecated
export class DefaultTechDocsCollator {
// (undocumented)
protected applyArgsToFormat(
format: string,
@@ -42,6 +45,21 @@ export class DefaultTechDocsCollator implements DocumentCollator {
readonly visibilityPermission: Permission;
}
// @public
export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
// (undocumented)
static fromConfig(
config: Config,
options: TechDocsCollatorFactoryOptions,
): DefaultTechDocsCollatorFactory;
// (undocumented)
getCollator(): Promise<Readable>;
// (undocumented)
readonly type: string;
// (undocumented)
readonly visibilityPermission: Permission;
}
// @public
export interface DocsBuildStrategy {
// (undocumented)
@@ -81,6 +99,17 @@ export type ShouldBuildParameters = {
entity: Entity;
};
// @public
export type TechDocsCollatorFactoryOptions = {
discovery: PluginEndpointDiscovery;
logger: Logger_2;
tokenManager: TokenManager;
locationTemplate?: string;
catalogClient?: CatalogApi;
parallelismLimit?: number;
legacyPathCasing?: boolean;
};
// @public
export type TechDocsCollatorOptions = {
discovery: PluginEndpointDiscovery;
plugins/techdocs-backend/package.json
+1
View File
@@ -56,6 +56,7 @@
},
"devDependencies": {
"@backstage/cli": "^0.14.1",
"@backstage/plugin-search-backend-node": "0.4.7",
"@backstage/test-utils": "^0.2.6",
"@types/dockerode": "^3.3.0",
"msw": "^0.35.0",
plugins/techdocs-backend/src/index.ts
+8 -2
View File
@@ -29,8 +29,14 @@ export type {
ShouldBuildParameters,
} from './service';
export { DefaultTechDocsCollator } from './search';
export type { TechDocsCollatorOptions } from './search';
export {
DefaultTechDocsCollator,
DefaultTechDocsCollatorFactory,
} from './search';
export type {
TechDocsCollatorFactoryOptions,
TechDocsCollatorOptions,
} from './search';
/**
* @deprecated Use directly from @backstage/techdocs-common
plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts
-12
View File
@@ -72,18 +72,6 @@ const expectedEntities: Entity[] = [
owner: 'someone',
},
},
{
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'test-entity',
description: 'The expected description',
},
spec: {
type: 'some-type',
lifecycle: 'experimental',
},
},
];
describe('DefaultTechDocsCollator with legacyPathCasing configuration', () => {
plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts
+3 -2
View File
@@ -24,7 +24,6 @@ import {
RELATION_OWNED_BY,
stringifyEntityRef,
} from '@backstage/catalog-model';
import { DocumentCollator } from '@backstage/search-common';
import fetch from 'node-fetch';
import unescape from 'lodash/unescape';
import { Logger } from 'winston';
@@ -69,8 +68,10 @@ type EntityInfo = {
* A search collator responsible for gathering and transforming TechDocs documents.
*
* @public
* @deprecated Upgrade to a more recent `@backstage/search-backend-node` and
* use `DefaultTechDocsCollatorFactory` instead.
*/
export class DefaultTechDocsCollator implements DocumentCollator {
export class DefaultTechDocsCollator {
public readonly type: string = 'techdocs';
public readonly visibilityPermission = catalogEntityReadPermission;
plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts
+245
View File
@@ -0,0 +1,245 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
getVoidLogger,
PluginEndpointDiscovery,
TokenManager,
} from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import { TestPipeline } from '@backstage/plugin-search-backend-node';
import { setupRequestMockHandlers } from '@backstage/test-utils';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import { Readable } from 'stream';
import { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory';
const logger = getVoidLogger();
const mockSearchDocIndex = {
config: {
lang: ['en'],
min_search_length: 3,
prebuild_index: false,
separator: '[\\s\\-]+',
},
docs: [
{
location: '',
text: 'docs docs docs',
title: 'Home',
},
{
location: 'local-development/',
text: 'Docs for first subtitle',
title: 'Local development',
},
{
location: 'local-development/#development',
text: 'Docs for sub-subtitle',
title: 'Development',
},
],
};
const expectedEntities: Entity[] = [
{
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
title: 'Test Entity with Docs!',
name: 'test-entity-with-docs',
description: 'Documented description',
annotations: {
'backstage.io/techdocs-ref': './',
},
},
spec: {
type: 'dog',
lifecycle: 'experimental',
owner: 'someone',
},
},
];
describe('DefaultTechDocsCollatorFactory', () => {
const config = new ConfigReader({});
const mockDiscoveryApi: jest.Mocked<PluginEndpointDiscovery> = {
getBaseUrl: jest.fn().mockResolvedValue('http://test-backend'),
getExternalBaseUrl: jest.fn(),
};
const mockTokenManager: jest.Mocked<TokenManager> = {
getToken: jest.fn().mockResolvedValue({ token: '' }),
authenticate: jest.fn(),
};
const options = {
discovery: mockDiscoveryApi,
logger: getVoidLogger(),
tokenManager: mockTokenManager,
};
it('has expected type', () => {
const factory = DefaultTechDocsCollatorFactory.fromConfig(config, options);
expect(factory.type).toBe('techdocs');
});
describe('getCollator', () => {
let factory: DefaultTechDocsCollatorFactory;
let collator: Readable;
const worker = setupServer();
setupRequestMockHandlers(worker);
beforeEach(async () => {
factory = DefaultTechDocsCollatorFactory.fromConfig(config, options);
collator = await factory.getCollator();
worker.use(
rest.get(
'http://test-backend/static/docs/default/Component/test-entity-with-docs/search/search_index.json',
(_, res, ctx) => res(ctx.status(200), ctx.json(mockSearchDocIndex)),
),
rest.get('http://test-backend/entities', (req, res, ctx) => {
// Imitate offset/limit pagination.
const offset = parseInt(
req.url.searchParams.get('offset') || '0',
10,
);
const limit = parseInt(
req.url.searchParams.get('limit') || '500',
10,
);
// Limit 50 corresponds to a case testing pagination.
if (limit === 50) {
// Return 50 copies of invalid entities on the first request.
if (offset === 0) {
return res(ctx.status(200), ctx.json(Array(50).fill({})));
}
// Then just the regular 2 on the second.
return res(ctx.status(200), ctx.json(expectedEntities));
}
return res(
ctx.status(200),
ctx.json(expectedEntities.slice(offset, limit + offset)),
);
}),
);
});
it('returns a readable stream', async () => {
expect(collator).toBeInstanceOf(Readable);
});
it('fetches from the configured catalog and tech docs services', async () => {
const pipeline = TestPipeline.withSubject(collator);
const { documents } = await pipeline.execute();
expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog');
expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('techdocs');
expect(documents).toHaveLength(mockSearchDocIndex.docs.length);
});
it('should create documents for each tech docs search index', async () => {
const pipeline = TestPipeline.withSubject(collator);
const { documents } = await pipeline.execute();
const entity = expectedEntities[0];
documents.forEach((document, idx) => {
expect(document).toMatchObject({
title: mockSearchDocIndex.docs[idx].title,
location: `/docs/default/component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`,
text: mockSearchDocIndex.docs[idx].text,
namespace: 'default',
entityTitle: entity!.metadata.title,
componentType: entity!.spec!.type,
lifecycle: entity!.spec!.lifecycle,
owner: '',
kind: entity.kind.toLocaleLowerCase('en-US'),
name: entity.metadata.name,
});
});
});
it('maps a returned entity with a custom locationTemplate', async () => {
// Provide an alternate location template.
factory = DefaultTechDocsCollatorFactory.fromConfig(config, {
discovery: mockDiscoveryApi,
tokenManager: mockTokenManager,
locationTemplate: '/software/:name',
logger,
});
collator = await factory.getCollator();
const pipeline = TestPipeline.withSubject(collator);
const { documents } = await pipeline.execute();
expect(documents[0]).toMatchObject({
location: '/software/test-entity-with-docs',
});
});
it('paginates through catalog entities using batchSize', async () => {
// A parallelismLimit of 1 is a catalog limit of 50 per request. Code
// above in the /entities handler ensures valid entities are only
// returned on the second page.
factory = DefaultTechDocsCollatorFactory.fromConfig(config, {
...options,
parallelismLimit: 1,
});
collator = await factory.getCollator();
const pipeline = TestPipeline.withSubject(collator);
const { documents } = await pipeline.execute();
// Only 1 entity with TechDocs configured multipled by 3 pages.
expect(documents).toHaveLength(3);
});
describe('with legacyPathCasing configuration', () => {
beforeEach(async () => {
const legacyConfig = new ConfigReader({
techdocs: {
legacyUseCaseSensitiveTripletPaths: true,
},
});
factory = DefaultTechDocsCollatorFactory.fromConfig(
legacyConfig,
options,
);
collator = await factory.getCollator();
});
it('should create documents for each tech docs search index', async () => {
const pipeline = TestPipeline.withSubject(collator);
const { documents } = await pipeline.execute();
const entity = expectedEntities[0];
documents.forEach((document, idx) => {
expect(document).toMatchObject({
title: mockSearchDocIndex.docs[idx].title,
location: `/docs/default/Component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`,
text: mockSearchDocIndex.docs[idx].text,
namespace: 'default',
entityTitle: entity!.metadata.title,
componentType: entity!.spec!.type,
lifecycle: entity!.spec!.lifecycle,
owner: '',
kind: entity.kind,
name: entity.metadata.name,
});
});
});
});
});
});
plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts
+254
View File
@@ -0,0 +1,254 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
PluginEndpointDiscovery,
TokenManager,
} from '@backstage/backend-common';
import {
CatalogApi,
CatalogClient,
CATALOG_FILTER_EXISTS,
} from '@backstage/catalog-client';
import {
Entity,
parseEntityRef,
RELATION_OWNED_BY,
stringifyEntityRef,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common';
import { DocumentCollatorFactory } from '@backstage/search-common';
import { TechDocsDocument } from '@backstage/techdocs-common';
import unescape from 'lodash/unescape';
import fetch from 'node-fetch';
import pLimit from 'p-limit';
import { Readable } from 'stream';
import { Logger } from 'winston';
interface MkSearchIndexDoc {
title: string;
text: string;
location: string;
}
/**
* Options to configure the TechDocs collator factory
*
* @public
*/
export type TechDocsCollatorFactoryOptions = {
discovery: PluginEndpointDiscovery;
logger: Logger;
tokenManager: TokenManager;
locationTemplate?: string;
catalogClient?: CatalogApi;
parallelismLimit?: number;
legacyPathCasing?: boolean;
};
type EntityInfo = {
name: string;
namespace: string;
kind: string;
};
/**
* A search collator factory responsible for gathering and transforming
* TechDocs documents.
*
* @public
*/
export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory {
public readonly type: string = 'techdocs';
public readonly visibilityPermission = catalogEntityReadPermission;
private discovery: PluginEndpointDiscovery;
private locationTemplate: string;
private readonly logger: Logger;
private readonly catalogClient: CatalogApi;
private readonly tokenManager: TokenManager;
private readonly parallelismLimit: number;
private readonly legacyPathCasing: boolean;
private constructor(options: TechDocsCollatorFactoryOptions) {
this.discovery = options.discovery;
this.locationTemplate =
options.locationTemplate || '/docs/:namespace/:kind/:name/:path';
this.logger = options.logger;
this.catalogClient =
options.catalogClient ||
new CatalogClient({ discoveryApi: options.discovery });
this.parallelismLimit = options.parallelismLimit ?? 10;
this.legacyPathCasing = options.legacyPathCasing ?? false;
this.tokenManager = options.tokenManager;
}
static fromConfig(config: Config, options: TechDocsCollatorFactoryOptions) {
const legacyPathCasing =
config.getOptionalBoolean(
'techdocs.legacyUseCaseSensitiveTripletPaths',
) || false;
return new DefaultTechDocsCollatorFactory({ ...options, legacyPathCasing });
}
async getCollator(): Promise<Readable> {
return Readable.from(this.execute());
}
private async *execute(): AsyncGenerator<TechDocsDocument, void, undefined> {
const limit = pLimit(this.parallelismLimit);
const techDocsBaseUrl = await this.discovery.getBaseUrl('techdocs');
const { token } = await this.tokenManager.getToken();
let entitiesRetrieved = 0;
let moreEntitiesToGet = true;
// Offset/limit pagination is used on the Catalog Client in order to
// limit (and allow some control over) memory used by the search backend
// at index-time. The batchSize is calculated as a factor of the given
// parallelism limit to simplify configuration.
const batchSize = this.parallelismLimit * 50;
while (moreEntitiesToGet) {
const entities = (
await this.catalogClient.getEntities(
{
filter: {
'metadata.annotations.backstage.io/techdocs-ref':
CATALOG_FILTER_EXISTS,
},
fields: [
'kind',
'namespace',
'metadata.annotations',
'metadata.name',
'metadata.title',
'metadata.namespace',
'spec.type',
'spec.lifecycle',
'relations',
],
limit: batchSize,
offset: entitiesRetrieved,
},
{ token },
)
).items;
// Control looping through entity batches.
moreEntitiesToGet = entities.length === batchSize;
entitiesRetrieved += entities.length;
const docPromises = entities
.filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref'])
.map((entity: Entity) =>
limit(async (): Promise<TechDocsDocument[]> => {
const entityInfo =
DefaultTechDocsCollatorFactory.handleEntityInfoCasing(
this.legacyPathCasing,
{
kind: entity.kind,
namespace: entity.metadata.namespace || 'default',
name: entity.metadata.name,
},
);
try {
const searchIndexResponse = await fetch(
DefaultTechDocsCollatorFactory.constructDocsIndexUrl(
techDocsBaseUrl,
entityInfo,
),
{
headers: {
Authorization: `Bearer ${token}`,
},
},
);
const searchIndex = await searchIndexResponse.json();
return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({
title: unescape(doc.title),
text: unescape(doc.text || ''),
location: this.applyArgsToFormat(
this.locationTemplate || '/docs/:namespace/:kind/:name/:path',
{
...entityInfo,
path: doc.location,
},
),
path: doc.location,
...entityInfo,
entityTitle: entity.metadata.title,
componentType: entity.spec?.type?.toString() || 'other',
lifecycle: (entity.spec?.lifecycle as string) || '',
owner: getSimpleEntityOwnerString(entity),
authorization: {
resourceRef: stringifyEntityRef(entity),
},
}));
} catch (e) {
this.logger.debug(
`Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`,
e,
);
return [];
}
}),
);
yield* (await Promise.all(docPromises)).flat();
}
}
private applyArgsToFormat(
format: string,
args: Record<string, string>,
): string {
let formatted = format;
for (const [key, value] of Object.entries(args)) {
formatted = formatted.replace(`:${key}`, value);
}
return formatted;
}
private static constructDocsIndexUrl(
techDocsBaseUrl: string,
entityInfo: { kind: string; namespace: string; name: string },
) {
return `${techDocsBaseUrl}/static/docs/${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}/search/search_index.json`;
}
private static handleEntityInfoCasing(
legacyPaths: boolean,
entityInfo: EntityInfo,
): EntityInfo {
return legacyPaths
? entityInfo
: Object.entries(entityInfo).reduce((acc, [key, value]) => {
return { ...acc, [key]: value.toLocaleLowerCase('en-US') };
}, {} as EntityInfo);
}
}
function getSimpleEntityOwnerString(entity: Entity): string {
if (entity.relations) {
const owner = entity.relations.find(r => r.type === RELATION_OWNED_BY);
if (owner) {
const { name } = parseEntityRef(owner.targetRef);
return name;
}
}
return '';
}
plugins/techdocs-backend/src/search/index.ts
+7 -1
View File
@@ -13,6 +13,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { DefaultTechDocsCollator } from './DefaultTechDocsCollator';
export { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory';
export type { TechDocsCollatorFactoryOptions } from './DefaultTechDocsCollatorFactory';
/**
* todo(backstage/techdocs-core): stop exporting these in a future release.
*/
export { DefaultTechDocsCollator } from './DefaultTechDocsCollator';
export type { TechDocsCollatorOptions } from './DefaultTechDocsCollator';