Merge remote-tracking branch 'upstream/master' into feat/techdocs-build-logs

Signed-off-by: Dominik Henneke <dominik.henneke@sda-se.com>
This commit is contained in:
Dominik Henneke
2021-07-13 16:15:08 +02:00
119 changed files with 9860 additions and 5109 deletions
@@ -18,9 +18,7 @@ import {
getVoidLogger,
PluginEndpointDiscovery,
} from '@backstage/backend-common';
import { CatalogApi } from '@backstage/catalog-client';
import { ConfigReader } from '@backstage/config';
import { NotFoundError } from '@backstage/errors';
import {
GeneratorBuilder,
PreparerBuilder,
@@ -29,12 +27,8 @@ import {
import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder';
import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer';
jest.mock('@backstage/config');
jest.mock('../DocsBuilder');
const MockedConfigReader = ConfigReader as jest.MockedClass<
typeof ConfigReader
>;
const MockedDocsBuilder = DocsBuilder as jest.MockedClass<typeof DocsBuilder>;
describe('DocsSynchronizer', () => {
@@ -57,16 +51,6 @@ describe('DocsSynchronizer', () => {
getBaseUrl: jest.fn(),
getExternalBaseUrl: jest.fn(),
};
const catalogClient: jest.Mocked<CatalogApi> = {
getLocationById: jest.fn(),
getEntityByName: jest.fn(),
getEntities: jest.fn(),
addLocation: jest.fn(),
removeLocationById: jest.fn(),
getOriginLocationByEntity: jest.fn(),
getLocationByEntity: jest.fn(),
removeEntityByUid: jest.fn(),
};
let docsSynchronizer: DocsSynchronizer;
const mockResponseHandler: jest.Mocked<DocsSynchronizerSyncOpts> = {
@@ -82,10 +66,7 @@ describe('DocsSynchronizer', () => {
});
docsSynchronizer = new DocsSynchronizer({
preparers,
generators,
publisher,
catalogClient,
config: new ConfigReader({}),
logger: getVoidLogger(),
});
@@ -98,7 +79,6 @@ describe('DocsSynchronizer', () => {
describe('doSync', () => {
it('should execute an update', async () => {
(shouldCheckForUpdate as jest.Mock).mockReturnValue(true);
MockedConfigReader.prototype.getString.mockReturnValue('local');
const entity = {
apiVersion: 'backstage.io/v1alpha1',
@@ -107,15 +87,9 @@ describe('DocsSynchronizer', () => {
uid: '0',
name: 'test',
namespace: 'default',
annotations: {
'sda.se/release-notes-location':
'github-releases:https://github.com/backstage/backstage',
},
},
};
catalogClient.getEntityByName.mockResolvedValue(entity);
MockedDocsBuilder.prototype.build.mockImplementation(async () => {
// extract the logStream from the constructor call
const logStream = MockedDocsBuilder.mock.calls[0][0].logStream;
@@ -123,37 +97,34 @@ describe('DocsSynchronizer', () => {
logStream?.write('Some log');
logStream?.write('Another log');
const logger = MockedDocsBuilder.mock.calls[0][0].logger;
logger.info('Some more log');
return true;
});
publisher.hasDocsBeenGenerated.mockResolvedValue(true);
await docsSynchronizer.doSync(() => mockResponseHandler, {
kind: 'Component',
namespace: 'default',
name: 'test',
token: undefined,
await docsSynchronizer.doSync({
responseHandler: mockResponseHandler,
entity,
preparers,
generators,
});
expect(catalogClient.getEntityByName).toBeCalledWith(
{
kind: 'Component',
namespace: 'default',
name: 'test',
},
{ token: undefined },
);
expect(mockResponseHandler.log).toBeCalledTimes(2);
expect(mockResponseHandler.log).toBeCalledTimes(3);
expect(mockResponseHandler.log).toBeCalledWith('Some log');
expect(mockResponseHandler.log).toBeCalledWith('Another log');
expect(mockResponseHandler.log).toBeCalledWith(
expect.stringMatching(/info.*Some more log/),
);
expect(mockResponseHandler.finish).toBeCalledWith({ updated: true });
expect(mockResponseHandler.error).toBeCalledTimes(0);
expect(shouldCheckForUpdate).toBeCalledTimes(1);
expect(MockedConfigReader.prototype.getString).toBeCalledTimes(1);
expect(DocsBuilder.prototype.build).toBeCalledTimes(1);
});
@@ -167,20 +138,14 @@ describe('DocsSynchronizer', () => {
uid: '0',
name: 'test',
namespace: 'default',
annotations: {
'sda.se/release-notes-location':
'github-releases:https://github.com/backstage/backstage',
},
},
};
catalogClient.getEntityByName.mockResolvedValue(entity);
await docsSynchronizer.doSync(() => mockResponseHandler, {
kind: 'Component',
namespace: 'default',
name: 'test',
token: undefined,
await docsSynchronizer.doSync({
responseHandler: mockResponseHandler,
entity,
preparers,
generators,
});
expect(mockResponseHandler.finish).toBeCalledWith({ updated: false });
@@ -189,50 +154,11 @@ describe('DocsSynchronizer', () => {
expect(mockResponseHandler.error).toBeCalledTimes(0);
expect(shouldCheckForUpdate).toBeCalledTimes(1);
expect(MockedConfigReader.prototype.getString).toBeCalledTimes(0);
expect(DocsBuilder.prototype.build).toBeCalledTimes(0);
});
it('should not check for an update without local builder', async () => {
(shouldCheckForUpdate as jest.Mock).mockReturnValue(true);
MockedConfigReader.prototype.getString.mockReturnValue('external');
const entity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
uid: '0',
name: 'test',
namespace: 'default',
annotations: {
'sda.se/release-notes-location':
'github-releases:https://github.com/backstage/backstage',
},
},
};
catalogClient.getEntityByName.mockResolvedValue(entity);
await docsSynchronizer.doSync(() => mockResponseHandler, {
kind: 'Component',
namespace: 'default',
name: 'test',
token: undefined,
});
expect(mockResponseHandler.finish).toBeCalledWith({ updated: false });
expect(mockResponseHandler.log).toBeCalledTimes(0);
expect(mockResponseHandler.error).toBeCalledTimes(0);
expect(shouldCheckForUpdate).toBeCalledTimes(1);
expect(MockedConfigReader.prototype.getString).toBeCalledTimes(1);
expect(DocsBuilder.prototype.build).toBeCalledTimes(0);
});
it('should forward build errors', async () => {
(shouldCheckForUpdate as jest.Mock).mockReturnValue(true);
MockedConfigReader.prototype.getString.mockReturnValue('local');
const entity = {
apiVersion: 'backstage.io/v1alpha1',
@@ -241,23 +167,17 @@ describe('DocsSynchronizer', () => {
uid: '0',
name: 'test',
namespace: 'default',
annotations: {
'sda.se/release-notes-location':
'github-releases:https://github.com/backstage/backstage',
},
},
};
catalogClient.getEntityByName.mockResolvedValue(entity);
const error = new Error('Some random error');
MockedDocsBuilder.prototype.build.mockRejectedValue(error);
await docsSynchronizer.doSync(() => mockResponseHandler, {
kind: 'Component',
namespace: 'default',
name: 'test',
token: undefined,
await docsSynchronizer.doSync({
responseHandler: mockResponseHandler,
entity,
preparers,
generators,
});
expect(mockResponseHandler.log).toBeCalledTimes(1);
@@ -270,22 +190,5 @@ describe('DocsSynchronizer', () => {
expect(mockResponseHandler.error).toBeCalledTimes(1);
expect(mockResponseHandler.error).toBeCalledWith(error);
});
it('rejects when entity is not found', async () => {
catalogClient.getEntityByName.mockResolvedValue(undefined);
await expect(
docsSynchronizer.doSync(() => mockResponseHandler, {
kind: 'Component',
namespace: 'default',
name: 'test',
token: undefined,
}),
).rejects.toThrowError(NotFoundError);
expect(mockResponseHandler.finish).toBeCalledTimes(0);
expect(mockResponseHandler.log).toBeCalledTimes(0);
expect(mockResponseHandler.error).toBeCalledTimes(0);
});
});
});
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { CatalogApi } from '@backstage/catalog-client';
import { Entity } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { NotFoundError } from '@backstage/errors';
import {
@@ -33,62 +33,35 @@ export type DocsSynchronizerSyncOpts = {
};
export class DocsSynchronizer {
private readonly preparers: PreparerBuilder;
private readonly generators: GeneratorBuilder;
private readonly publisher: PublisherBase;
private readonly logger: winston.Logger;
private readonly config: Config;
private readonly catalogClient: CatalogApi;
constructor({
preparers,
generators,
publisher,
logger,
config,
catalogClient,
}: {
preparers: PreparerBuilder;
generators: GeneratorBuilder;
publisher: PublisherBase;
logger: winston.Logger;
config: Config;
catalogClient: CatalogApi;
}) {
this.catalogClient = catalogClient;
this.config = config;
this.logger = logger;
this.publisher = publisher;
this.generators = generators;
this.preparers = preparers;
}
async doSync(
initResponseHandler: () => DocsSynchronizerSyncOpts,
{
kind,
namespace,
name,
token,
}: {
kind: string;
namespace: string;
name: string;
token: string | undefined;
},
) {
const entity = await this.catalogClient.getEntityByName(
{ kind, namespace, name },
{ token },
);
if (!entity?.metadata?.uid) {
throw new NotFoundError('Entity metadata UID missing');
}
// open the event-stream
const { log, error, finish } = initResponseHandler();
async doSync({
responseHandler: { log, error, finish },
entity,
preparers,
generators,
}: {
responseHandler: DocsSynchronizerSyncOpts;
entity: Entity;
preparers: PreparerBuilder;
generators: GeneratorBuilder;
}) {
// create a new logger to log data to the caller
const taskLogger = winston.createLogger({
level: process.env.LOG_LEVEL || 'info',
@@ -109,22 +82,14 @@ export class DocsSynchronizer {
taskLogger.add(new winston.transports.Stream({ stream: logStream }));
// check if the last update check was too recent
if (!shouldCheckForUpdate(entity.metadata.uid)) {
finish({ updated: false });
return;
}
// techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to 'local'
// If set to 'external', it will assume that an external process (e.g. CI/CD pipeline
// of the repository) is responsible for building and publishing documentation to the storage provider
if (this.config.getString('techdocs.builder') !== 'local') {
if (!shouldCheckForUpdate(entity.metadata.uid!)) {
finish({ updated: false });
return;
}
const docsBuilder = new DocsBuilder({
preparers: this.preparers,
generators: this.generators,
preparers,
generators,
publisher: this.publisher,
logger: taskLogger,
entity,
@@ -19,8 +19,9 @@ import {
getVoidLogger,
PluginEndpointDiscovery,
} from '@backstage/backend-common';
import { CatalogClient } from '@backstage/catalog-client';
import { ConfigReader } from '@backstage/config';
import { NotFoundError, NotModifiedError } from '@backstage/errors';
import { NotModifiedError } from '@backstage/errors';
import {
GeneratorBuilder,
PreparerBuilder,
@@ -31,13 +32,37 @@ import request from 'supertest';
import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer';
import { createEventStream, createHttpResponse, createRouter } from './router';
jest.mock('@backstage/catalog-client');
jest.mock('@backstage/config');
jest.mock('./DocsSynchronizer');
const MockedConfigReader = ConfigReader as jest.MockedClass<
typeof ConfigReader
>;
const MockCatalogClient = CatalogClient as jest.MockedClass<
typeof CatalogClient
>;
const MockDocsSynchronizer = DocsSynchronizer as jest.MockedClass<
typeof DocsSynchronizer
>;
describe('createRouter', () => {
const entity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
uid: '0',
name: 'test',
},
};
const entityWithoutMetadata = {
...entity,
metadata: {
...entity.metadata,
uid: undefined,
},
};
const preparers: jest.Mocked<PreparerBuilder> = {
register: jest.fn(),
get: jest.fn(),
@@ -70,7 +95,7 @@ describe('createRouter', () => {
return `http://backstage.local/api/${type}`;
});
const router = await createRouter({
const outOfTheBoxRouter = await createRouter({
preparers,
generators,
publisher,
@@ -78,37 +103,101 @@ describe('createRouter', () => {
logger: getVoidLogger(),
discovery,
});
const recommendedRouter = await createRouter({
publisher,
config: new ConfigReader({}),
logger: getVoidLogger(),
discovery,
});
router.use(errorHandler());
app = express();
app.use(router);
app.use(outOfTheBoxRouter);
app.use('/recommended', recommendedRouter);
app.use(errorHandler());
});
describe('GET /sync/:namespace/:kind/:name', () => {
describe('accept application/json', () => {
it('should return not found if entity is not found', async () => {
MockCatalogClient.prototype.getEntityByName.mockResolvedValue(
undefined,
);
const response = await request(app)
.get('/sync/default/Component/test')
.send();
expect(response.status).toBe(404);
});
it('should return not found if entity has no uid', async () => {
MockCatalogClient.prototype.getEntityByName.mockResolvedValue(
entityWithoutMetadata,
);
const response = await request(app)
.get('/sync/default/Component/test')
.send();
expect(response.status).toBe(404);
});
it('should not check for an update without local builder', async () => {
MockedConfigReader.prototype.getString.mockReturnValue('external');
MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity);
const response = await request(app)
.get('/sync/default/Component/test')
.send();
expect(response.status).toBe(304);
});
it('should error if missing builder', async () => {
MockedConfigReader.prototype.getString.mockReturnValue('local');
MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity);
const response = await request(app)
.get('/recommended/sync/default/Component/test')
.send();
expect(response.status).toBe(500);
expect(response.text).toMatch(
/Invalid configuration\. 'techdocs\.builder' was set to 'local' but no 'preparer' was provided to the router initialization/,
);
expect(MockDocsSynchronizer.prototype.doSync).toBeCalledTimes(0);
});
it('should execute synchronization', async () => {
MockedConfigReader.prototype.getString.mockReturnValue('local');
MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity);
MockDocsSynchronizer.prototype.doSync.mockImplementation(
async handler => handler().finish({ updated: true }),
async ({ responseHandler }) =>
responseHandler.finish({ updated: true }),
);
await request(app).get('/sync/default/Component/test').send();
expect(MockDocsSynchronizer.prototype.doSync).toBeCalledTimes(1);
expect(MockDocsSynchronizer.prototype.doSync).toBeCalledWith(
expect.any(Function),
{
kind: 'Component',
name: 'test',
namespace: 'default',
token: undefined,
expect(MockDocsSynchronizer.prototype.doSync).toBeCalledWith({
responseHandler: {
log: expect.any(Function),
error: expect.any(Function),
finish: expect.any(Function),
},
);
entity,
generators,
preparers,
});
});
it('should return on updated', async () => {
MockedConfigReader.prototype.getString.mockReturnValue('local');
MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity);
MockDocsSynchronizer.prototype.doSync.mockImplementation(
async handler => {
const { log, finish } = handler();
async ({ responseHandler }) => {
const { log, finish } = responseHandler;
log('Some log');
@@ -126,43 +215,81 @@ describe('createRouter', () => {
'{"message":"Docs updated or did not need updating"}',
);
});
});
it('should return error', async () => {
MockDocsSynchronizer.prototype.doSync.mockImplementation(
async handler => {
const { log, error } = handler();
log('Some log');
error(new Error('Some Error'));
},
);
const response = await request(app)
.get('/sync/default/Component/test')
.send();
expect(response.status).toBe(500);
expect(response.text).toMatch(/Some Error/);
});
it('should return not found', async () => {
MockDocsSynchronizer.prototype.doSync.mockRejectedValue(
new NotFoundError(),
describe('accept text/event-stream', () => {
it('should return not found if entity is not found', async () => {
MockCatalogClient.prototype.getEntityByName.mockResolvedValue(
undefined,
);
const response = await request(app)
.get('/sync/default/Component/test')
.set('accept', 'text/event-stream')
.send();
expect(response.status).toBe(404);
});
});
describe('accept text/event-stream', () => {
it('should return not found if entity has no uid', async () => {
MockCatalogClient.prototype.getEntityByName.mockResolvedValue(
entityWithoutMetadata,
);
const response = await request(app)
.get('/sync/default/Component/test')
.set('accept', 'text/event-stream')
.send();
expect(response.status).toBe(404);
});
it('should not check for an update without local builder', async () => {
MockedConfigReader.prototype.getString.mockReturnValue('external');
MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity);
const response = await request(app)
.get('/sync/default/Component/test')
.set('accept', 'text/event-stream')
.send();
expect(response.status).toBe(200);
expect(response.get('content-type')).toBe('text/event-stream');
expect(response.text).toEqual(
`event: finish
data: {"updated":false}
`,
);
});
it('should error if missing builder', async () => {
MockedConfigReader.prototype.getString.mockReturnValue('local');
MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity);
const response = await request(app)
.get('/recommended/sync/default/Component/test')
.set('accept', 'text/event-stream')
.send();
expect(response.status).toBe(200);
expect(response.get('content-type')).toBe('text/event-stream');
expect(response.text).toEqual(
`event: error
data: "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'preparer' was provided to the router initialization."
`,
);
expect(MockDocsSynchronizer.prototype.doSync).toBeCalledTimes(0);
});
it('should execute synchronization', async () => {
MockedConfigReader.prototype.getString.mockReturnValue('local');
MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity);
MockDocsSynchronizer.prototype.doSync.mockImplementation(
async handler => handler().finish({ updated: true }),
async ({ responseHandler }) =>
responseHandler.finish({ updated: true }),
);
await request(app)
@@ -171,21 +298,24 @@ describe('createRouter', () => {
.send();
expect(MockDocsSynchronizer.prototype.doSync).toBeCalledTimes(1);
expect(MockDocsSynchronizer.prototype.doSync).toBeCalledWith(
expect.any(Function),
{
kind: 'Component',
name: 'test',
namespace: 'default',
token: undefined,
expect(MockDocsSynchronizer.prototype.doSync).toBeCalledWith({
responseHandler: {
log: expect.any(Function),
error: expect.any(Function),
finish: expect.any(Function),
},
);
entity,
generators,
preparers,
});
});
it('should return an event-stream', async () => {
MockedConfigReader.prototype.getString.mockReturnValue('local');
MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity);
MockDocsSynchronizer.prototype.doSync.mockImplementation(
async handler => {
const { log, finish } = handler();
async ({ responseHandler }) => {
const { log, finish } = responseHandler;
log('Some log');
log('Another log');
@@ -214,48 +344,6 @@ data: {"updated":true}
`,
);
});
it('should return error', async () => {
MockDocsSynchronizer.prototype.doSync.mockImplementation(
async handler => {
const { log, error } = handler();
log('Some log');
error(new Error('Some Error'));
},
);
const response = await request(app)
.get('/sync/default/Component/test')
.set('accept', 'text/event-stream')
.send();
expect(response.status).toBe(200);
expect(response.get('content-type')).toBe('text/event-stream');
expect(response.text).toEqual(
`event: log
data: "Some log"
event: error
data: "Some Error"
`,
);
});
it('should return not found', async () => {
MockDocsSynchronizer.prototype.doSync.mockRejectedValue(
new NotFoundError(),
);
const response = await request(app)
.get('/sync/default/Component/test')
.set('accept', 'text/event-stream')
.send();
expect(response.status).toBe(404);
});
});
});
});
+79 -28
View File
@@ -31,7 +31,11 @@ import { Knex } from 'knex';
import { Logger } from 'winston';
import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer';
type RouterOptions = {
/**
* All of the required dependencies for running TechDocs in the "out-of-the-box"
* deployment configuration (prepare/generate/publish all in the Backend).
*/
type OutOfTheBoxDeploymentOptions = {
preparers: PreparerBuilder;
generators: GeneratorBuilder;
publisher: PublisherBase;
@@ -41,23 +45,44 @@ type RouterOptions = {
config: Config;
};
export async function createRouter({
preparers,
generators,
publisher,
config,
logger,
discovery,
}: RouterOptions): Promise<express.Router> {
/**
* Required dependencies for running TechDocs in the "recommended" deployment
* configuration (prepare/generate handled externally in CI/CD).
*/
type RecommendedDeploymentOptions = {
publisher: PublisherBase;
logger: Logger;
discovery: PluginEndpointDiscovery;
config: Config;
};
/**
* One of the two deployment configurations must be provided.
*/
type RouterOptions =
| RecommendedDeploymentOptions
| OutOfTheBoxDeploymentOptions;
/**
* Typeguard to help createRouter() understand when we are in a "recommended"
* deployment vs. when we are in an out-of-the-box deployment configuration.
*/
function isOutOfTheBoxOption(
opt: RouterOptions,
): opt is OutOfTheBoxDeploymentOptions {
return (opt as OutOfTheBoxDeploymentOptions).preparers !== undefined;
}
export async function createRouter(
options: RouterOptions,
): Promise<express.Router> {
const router = Router();
const { publisher, config, logger, discovery } = options;
const catalogClient = new CatalogClient({ discoveryApi: discovery });
const docsSynchronizer = new DocsSynchronizer({
preparers: preparers,
generators: generators,
publisher: publisher,
logger: logger,
config: config,
catalogClient: catalogClient,
});
router.get('/metadata/techdocs/:namespace/:kind/:name', async (req, res) => {
@@ -124,23 +149,49 @@ export async function createRouter({
const { kind, namespace, name } = req.params;
const token = getBearerToken(req.headers.authorization);
await docsSynchronizer.doSync(
() => {
if (req.header('accept') !== 'text/event-stream') {
console.warn(
"The call to /sync/:namespace/:kind/:name wasn't done by an EventSource. This behavior is deprecated and will be removed soon. Make sure to update the @backstage/plugin-techdocs package in the frontend to the latest version.",
);
return createHttpResponse(res);
}
const entity = await catalogClient.getEntityByName(
{ kind, namespace, name },
{ token },
);
return createEventStream(res);
},
{
kind,
namespace,
name,
token,
},
if (!entity?.metadata?.uid) {
throw new NotFoundError('Entity metadata UID missing');
}
let responseHandler: DocsSynchronizerSyncOpts;
if (req.header('accept') !== 'text/event-stream') {
console.warn(
"The call to /sync/:namespace/:kind/:name wasn't done by an EventSource. This behavior is deprecated and will be removed soon. Make sure to update the @backstage/plugin-techdocs package in the frontend to the latest version.",
);
responseHandler = createHttpResponse(res);
} else {
responseHandler = createEventStream(res);
}
// techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to 'local'
// If set to 'external', it will assume that an external process (e.g. CI/CD pipeline
// of the repository) is responsible for building and publishing documentation to the storage provider
if (config.getString('techdocs.builder') !== 'local') {
responseHandler.finish({ updated: false });
return;
}
if (isOutOfTheBoxOption(options)) {
const { preparers, generators } = options;
await docsSynchronizer.doSync({
responseHandler,
entity,
preparers,
generators,
});
return;
}
responseHandler.error(
new Error(
"Invalid configuration. 'techdocs.builder' was set to 'local' but no 'preparer' was provided to the router initialization.",
),
);
});