paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 14 Packages Projects Releases Wiki Activity

Rewrite the /sync/:namespace/:kind/:name to return an event-stream

Browse Source 
Signed-off-by: Dominik Henneke <dominik.henneke@sda-se.com>
This commit is contained in:
Dominik Henneke
2021-07-05 12:43:23 +02:00
parent c18e8eb910
commit f1200f44c8
15 changed files with 691 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/techdocs-backend/src/DocsBuilder/builder.ts
+8 -1
View File
@@ -18,6 +18,7 @@ import {
ENTITY_DEFAULT_NAMESPACE,
stringifyEntityRef,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { NotModifiedError } from '@backstage/errors';
import {
GeneratorBase,
@@ -31,8 +32,8 @@ import {
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
import { Writable } from 'stream';
import { Logger } from 'winston';
import { Config } from '@backstage/config';
import { BuildMetadataStorage } from './BuildMetadataStorage';
type DocsBuilderArguments = {
@@ -42,6 +43,7 @@ type DocsBuilderArguments = {
entity: Entity;
logger: Logger;
config: Config;
logStream?: Writable;
};
export class DocsBuilder {
@@ -51,6 +53,7 @@ export class DocsBuilder {
private entity: Entity;
private logger: Logger;
private config: Config;
private logStream: Writable | undefined;
constructor({
preparers,
@@ -59,6 +62,7 @@ export class DocsBuilder {
entity,
logger,
config,
logStream,
}: DocsBuilderArguments) {
this.preparer = preparers.get(entity);
this.generator = generators.get(entity);
@@ -66,6 +70,7 @@ export class DocsBuilder {
this.entity = entity;
this.logger = logger;
this.config = config;
this.logStream = logStream;
}
/**
@@ -159,12 +164,14 @@ export class DocsBuilder {
const outputDir = await fs.mkdtemp(
path.join(tmpdirResolvedPath, 'techdocs-tmp-'),
);
const parsedLocationAnnotation = getLocationForEntity(this.entity);
await this.generator.run({
inputDir: preparedDir,
outputDir,
parsedLocationAnnotation,
etag: newEtag,
logStream: this.logStream,
});
// Remove Prepared directory since it is no longer needed.
plugins/techdocs-backend/src/service/router.test.ts
+267
View File
@@ -0,0 +1,267 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
errorHandler,
getVoidLogger,
PluginEndpointDiscovery,
} from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import {
GeneratorBuilder,
PreparerBuilder,
PublisherBase,
} from '@backstage/techdocs-common';
import express from 'express';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import request from 'supertest';
import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder';
import { createRouter } from './router';
jest.mock('@backstage/config');
jest.mock('../DocsBuilder');
const MockedConfigReader = ConfigReader as jest.MockedClass<
typeof ConfigReader
>;
const MockedDocsBuilder = DocsBuilder as jest.MockedClass<typeof DocsBuilder>;
const server = setupServer();
describe('createRouter', () => {
// the calls from supertest should not be handled by msw so we only warn onUnhandledRequest
beforeAll(() => server.listen({ onUnhandledRequest: 'warn' }));
afterAll(() => server.close());
afterEach(() => server.resetHandlers());
const preparers: jest.Mocked<PreparerBuilder> = {
register: jest.fn(),
get: jest.fn(),
};
const generators: jest.Mocked<GeneratorBuilder> = {
register: jest.fn(),
get: jest.fn(),
};
const publisher: jest.Mocked<PublisherBase> = {
docsRouter: jest.fn(),
fetchTechDocsMetadata: jest.fn(),
getReadiness: jest.fn(),
hasDocsBeenGenerated: jest.fn(),
publish: jest.fn(),
};
const discovery: jest.Mocked<PluginEndpointDiscovery> = {
getBaseUrl: jest.fn(),
getExternalBaseUrl: jest.fn(),
};
let app: express.Express;
beforeEach(() => {
jest.resetAllMocks();
});
beforeEach(async () => {
publisher.docsRouter.mockReturnValue(() => {});
discovery.getBaseUrl.mockImplementation(async type => {
return `http://backstage.local/api/${type}`;
});
const router = await createRouter({
preparers,
generators,
publisher,
config: new ConfigReader({}),
logger: getVoidLogger(),
discovery,
});
router.use(errorHandler());
app = express();
app.use(router);
});
describe('GET /sync/:namespace/:kind/:name', () => {
it('should execute an update', async () => {
(shouldCheckForUpdate as jest.Mock).mockReturnValue(true);
MockedConfigReader.prototype.getString.mockReturnValue('local');
const entity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
uid: '0',
name: 'test',
namespace: 'default',
annotations: {
'sda.se/release-notes-location':
'github-releases:https://github.com/backstage/backstage',
},
},
};
server.use(
rest.get(
'http://backstage.local/api/catalog/entities/by-name/Component/default/test',
(_req, res, ctx) => {
return res(ctx.json(entity));
},
),
);
MockedDocsBuilder.prototype.build.mockImplementation(async () => {
// extract the logStream from the constructor call
const logStream = MockedDocsBuilder.mock.calls[0][0].logStream;
logStream?.write('Some log');
logStream?.write('Another log');
return true;
});
publisher.hasDocsBeenGenerated.mockResolvedValue(true);
const response = await request(app)
.get('/sync/default/Component/test')
.send();
expect(response.status).toBe(200);
expect(response.get('content-type')).toBe('text/event-stream');
expect(response.text).toEqual(
`event: log
data: "Some log"
event: log
data: "Another log"
event: finish
data: {"updated":true}
`,
);
expect(shouldCheckForUpdate).toBeCalledTimes(1);
expect(MockedConfigReader.prototype.getString).toBeCalledTimes(1);
expect(DocsBuilder.prototype.build).toBeCalledTimes(1);
});
it('should not check for an update too often', async () => {
(shouldCheckForUpdate as jest.Mock).mockReturnValue(false);
const entity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
uid: '0',
name: 'test',
namespace: 'default',
annotations: {
'sda.se/release-notes-location':
'github-releases:https://github.com/backstage/backstage',
},
},
};
server.use(
rest.get(
'http://backstage.local/api/catalog/entities/by-name/Component/default/test',
(_req, res, ctx) => {
return res(ctx.json(entity));
},
),
);
const response = await request(app)
.get('/sync/default/Component/test')
.send();
expect(response.status).toBe(200);
expect(response.get('content-type')).toBe('text/event-stream');
expect(response.text).toEqual(
`event: finish
data: {"updated":false}
`,
);
expect(shouldCheckForUpdate).toBeCalledTimes(1);
expect(MockedConfigReader.prototype.getString).toBeCalledTimes(0);
expect(DocsBuilder.prototype.build).toBeCalledTimes(0);
});
it('should not check for an update without local builder', async () => {
(shouldCheckForUpdate as jest.Mock).mockReturnValue(true);
MockedConfigReader.prototype.getString.mockReturnValue('external');
const entity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
uid: '0',
name: 'test',
namespace: 'default',
annotations: {
'sda.se/release-notes-location':
'github-releases:https://github.com/backstage/backstage',
},
},
};
server.use(
rest.get(
'http://backstage.local/api/catalog/entities/by-name/Component/default/test',
(_req, res, ctx) => {
return res(ctx.json(entity));
},
),
);
const response = await request(app)
.get('/sync/default/Component/test')
.send();
expect(response.status).toBe(200);
expect(response.get('content-type')).toBe('text/event-stream');
expect(response.text).toEqual(
`event: finish
data: {"updated":false}
`,
);
expect(shouldCheckForUpdate).toBeCalledTimes(1);
expect(MockedConfigReader.prototype.getString).toBeCalledTimes(1);
expect(DocsBuilder.prototype.build).toBeCalledTimes(0);
});
it('rejects when entity is not found', async () => {
server.use(
rest.get(
'http://backstage.local/api/catalog/entities/by-name/Component/default/test',
(_req, res, ctx) => {
return res(ctx.status(404));
},
),
);
const response = await request(app)
.get('/sync/default/Component/test')
.send();
expect(response.status).toBe(404);
});
});
});
plugins/techdocs-backend/src/service/router.ts
+107 -74
View File
@@ -14,9 +14,10 @@
* limitations under the License.
*/
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { CatalogClient } from '@backstage/catalog-client';
import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { NotFoundError, NotModifiedError } from '@backstage/errors';
import { NotFoundError } from '@backstage/errors';
import {
GeneratorBuilder,
getLocationForEntity,
@@ -24,12 +25,12 @@ import {
PublisherBase,
} from '@backstage/techdocs-common';
import fetch from 'cross-fetch';
import express from 'express';
import express, { Response } from 'express';
import Router from 'express-promise-router';
import { Knex } from 'knex';
import { PassThrough } from 'stream';
import { Logger } from 'winston';
import { DocsBuilder } from '../DocsBuilder';
import { shouldCheckForUpdate } from '../DocsBuilder/BuildMetadataStorage';
import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder';
type RouterOptions = {
preparers: PreparerBuilder;
@@ -50,6 +51,7 @@ export async function createRouter({
discovery,
}: RouterOptions): Promise<express.Router> {
const router = Router();
const catalogClient = new CatalogClient({ discoveryApi: discovery });
router.get('/metadata/techdocs/:namespace/:kind/:name', async (req, res) => {
const { kind, namespace, name } = req.params;
@@ -108,56 +110,45 @@ export async function createRouter({
});
// Check if docs are the latest version and trigger rebuilds if not
// Responds with immediate success if rebuild not needed
// Responds with an event-stream that closes after the build finished
// Responds with an immediate success if rebuild not needed
// If a build is required, responds with a success when finished
router.get('/sync/:namespace/:kind/:name', async (req, res) => {
const { kind, namespace, name } = req.params;
const catalogUrl = await discovery.getBaseUrl('catalog');
const triple = [kind, namespace, name].map(encodeURIComponent).join('/');
const token = getBearerToken(req.headers.authorization);
const catalogRes = await fetch(`${catalogUrl}/entities/by-name/${triple}`, {
headers: token ? { Authorization: `Bearer ${token}` } : {},
});
if (!catalogRes.ok) {
const catalogResText = await catalogRes.text();
res.status(catalogRes.status);
res.send(catalogResText);
return;
}
const entity: Entity = await catalogRes.json();
const entity = await catalogClient.getEntityByName(
{ kind, namespace, name },
{ token },
);
if (!entity.metadata.uid) {
if (!entity?.metadata?.uid) {
throw new NotFoundError('Entity metadata UID missing');
}
// open the event-stream
const { log, error, finish } = createEventStream(res);
// create an in-memory stream to forward logs to the event-stream
const logStream = new PassThrough();
logStream.on('data', async data => {
log(data.toString().trim());
});
// check if the last update check was too recent
if (!shouldCheckForUpdate(entity.metadata.uid)) {
res.status(200).json({
message: `Last check for documentation update is recent, did not retry.`,
});
finish({ updated: false });
return;
}
let publisherType = '';
try {
publisherType = config.getString('techdocs.publisher.type');
} catch (err) {
throw new Error(
'Unable to get techdocs.publisher.type in your app config. Set it to either ' +
"'local', 'googleGcs' or other support storage providers. Read more here " +
'https://backstage.io/docs/features/techdocs/architecture',
);
}
// techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to 'local'
// If set to 'external', it will assume that an external process (e.g. CI/CD pipeline
// of the repository) is responsible for building and publishing documentation to the storage provider
if (config.getString('techdocs.builder') !== 'local') {
res.status(200).json({
message:
'`techdocs.builder` app config is not set to `local`, so docs will not be generated locally and sync is not required.',
});
finish({ updated: false });
return;
}
const docsBuilder = new DocsBuilder({
preparers,
generators,
@@ -165,52 +156,41 @@ export async function createRouter({
logger,
entity,
config,
logStream,
});
let foundDocs = false;
switch (publisherType) {
case 'local':
case 'awsS3':
case 'azureBlobStorage':
case 'openStackSwift':
case 'googleGcs': {
// This block should be valid for all storage implementations. So no need to duplicate in future,
// add the publisher type in the list here.
const updated = await docsBuilder.build();
if (!updated) {
throw new NotModifiedError();
}
const updated = await docsBuilder.build();
// With a maximum of ~5 seconds wait, check if the files got published and if docs will be fetched
// on the user's page. If not, respond with a message asking them to check back later.
// The delay here is to make sure GCS/AWS/etc. registers newly uploaded files which is usually <1 second
for (let attempt = 0; attempt < 5; attempt++) {
if (await publisher.hasDocsBeenGenerated(entity)) {
foundDocs = true;
break;
}
await new Promise(r => setTimeout(r, 1000));
}
if (!foundDocs) {
logger.error(
'Published files are taking longer to show up in storage. Something went wrong.',
);
throw new NotFoundError(
'Sorry! It took too long for the generated docs to show up in storage. Check back later.',
);
}
if (!updated) {
finish({ updated: false });
return;
}
res
.status(201)
.json({ message: 'Docs updated or did not need updating' });
// With a maximum of ~5 seconds wait, check if the files got published and if docs will be fetched
// on the user's page. If not, respond with a message asking them to check back later.
// The delay here is to make sure GCS/AWS/etc. registers newly uploaded files which is usually <1 second
for (let attempt = 0; attempt < 5; attempt++) {
if (await publisher.hasDocsBeenGenerated(entity)) {
foundDocs = true;
break;
}
default:
throw new NotFoundError(
`Publisher type ${publisherType} is not supported by techdocs-backend docs builder.`,
);
await new Promise(r => setTimeout(r, 1000));
}
if (!foundDocs) {
logger.error(
'Published files are taking longer to show up in storage. Something went wrong.',
);
error(
new NotFoundError(
'Sorry! It took too long for the generated docs to show up in storage. Check back later.',
),
);
return;
}
finish({ updated: true });
});
// Route middleware which serves files from the storage set in the publisher.
@@ -222,3 +202,56 @@ export async function createRouter({
function getBearerToken(header?: string): string | undefined {
return header?.match(/(?:Bearer)\s+(\S+)/i)?.[1];
}
/**
* Create an event-stream response that emits the events 'log', 'error', and 'finish'.
*
* @param res the response to write the event-stream to
* @returns A tuple of <log, error, finish> callbacks to emit messages. A call to 'error' or 'finish'
* will close the event-stream.
*/
function createEventStream(
res: Response<any, any>,
): {
log: (message: string) => void;
error: (e: Error) => void;
finish: (result: { updated: boolean }) => void;
} {
// Mandatory headers and http status to keep connection open
res.writeHead(200, {
Connection: 'keep-alive',
'Cache-Control': 'no-cache',
'Content-Type': 'text/event-stream',
});
// client closes connection
res.socket?.on('close', () => {
res.end();
});
// write the event to the stream
const send = (type: 'error' | 'finish' | 'log', data: any) => {
res.write(`event: ${type}\ndata: ${JSON.stringify(data)}\n\n`);
// res.flush() is only available with the compression middleware
if (res.flush) {
res.flush();
}
};
return {
log: data => {
send('log', data);
},
error: e => {
send('error', e.message);
res.end();
},
finish: result => {
send('finish', result);
res.end();
},
};
}