diff --git a/.changeset/techdocs-few-balloons-lie.md b/.changeset/techdocs-few-balloons-lie.md new file mode 100644 index 0000000000..4e94c117ac --- /dev/null +++ b/.changeset/techdocs-few-balloons-lie.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Show a "Refresh" button to if the content is stale. +This removes the need to do a full page-reload to display more recent TechDocs content. diff --git a/.changeset/techdocs-loud-spies-attack.md b/.changeset/techdocs-loud-spies-attack.md new file mode 100644 index 0000000000..50fbe82fbc --- /dev/null +++ b/.changeset/techdocs-loud-spies-attack.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-techdocs': patch +'@backstage/plugin-techdocs-backend': patch +--- + +Rewrite the `/sync/:namespace/:kind/:name` endpoint to support an event-stream as response. +This change allows the sync process to take longer than a normal HTTP timeout. +The stream also emits log events, so the caller can follow the build process in the frontend. diff --git a/.changeset/techdocs-lovely-bottles-beam.md b/.changeset/techdocs-lovely-bottles-beam.md new file mode 100644 index 0000000000..3f89396c79 --- /dev/null +++ b/.changeset/techdocs-lovely-bottles-beam.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Provide a Drawer component to follow a running build. +This can be used to debug the rendering and get build logs in case an error occurs. diff --git a/.changeset/techdocs-new-candles-sell.md b/.changeset/techdocs-new-candles-sell.md new file mode 100644 index 0000000000..05443c9dbd --- /dev/null +++ b/.changeset/techdocs-new-candles-sell.md @@ -0,0 +1,6 @@ +--- +'@backstage/techdocs-common': patch +--- + +Provide optional `logger: Logger` and `logStream: Writable` arguments to the `GeneratorBase#run(...)` command. +They receive all log messages that are emitted during the generator run. diff --git a/packages/techdocs-common/api-report.md b/packages/techdocs-common/api-report.md index f3cfd29bcd..7ccb296f5f 100644 --- a/packages/techdocs-common/api-report.md +++ b/packages/techdocs-common/api-report.md @@ -55,6 +55,16 @@ export type GeneratorBuilder = { get(entity: Entity): GeneratorBase; }; +// @public +export type GeneratorRunOptions = { + inputDir: string; + outputDir: string; + parsedLocationAnnotation?: ParsedLocationAnnotation; + etag?: string; + logger: Logger_2; + logStream?: Writable; +}; + // @public (undocumented) export class Generators implements GeneratorBuilder { // (undocumented) @@ -232,6 +242,8 @@ export class TechdocsGenerator implements GeneratorBase { outputDir, parsedLocationAnnotation, etag, + logger: childLogger, + logStream, }: GeneratorRunOptions): Promise; } diff --git a/packages/techdocs-common/src/stages/generate/index.ts b/packages/techdocs-common/src/stages/generate/index.ts index 382712ccbc..eb015a5c2d 100644 --- a/packages/techdocs-common/src/stages/generate/index.ts +++ b/packages/techdocs-common/src/stages/generate/index.ts @@ -15,4 +15,8 @@ */ export { TechdocsGenerator } from './techdocs'; export { Generators } from './generators'; -export type { GeneratorBuilder, GeneratorBase } from './types'; +export type { + GeneratorBuilder, + GeneratorBase, + GeneratorRunOptions, +} from './types'; diff --git a/packages/techdocs-common/src/stages/generate/techdocs.ts b/packages/techdocs-common/src/stages/generate/techdocs.ts index 041dff1d2f..46ffc022fc 100644 --- a/packages/techdocs-common/src/stages/generate/techdocs.ts +++ b/packages/techdocs-common/src/stages/generate/techdocs.ts @@ -17,7 +17,6 @@ import { ContainerRunner } from '@backstage/backend-common'; import { Config } from '@backstage/config'; import path from 'path'; -import { PassThrough } from 'stream'; import { Logger } from 'winston'; import { addBuildTimestampMetadata, @@ -35,18 +34,6 @@ type TechdocsGeneratorOptions = { runGeneratorIn: string; }; -const createStream = (): [string[], PassThrough] => { - const log = [] as Array; - - const stream = new PassThrough(); - stream.on('data', chunk => { - const textValue = chunk.toString().trim(); - if (textValue?.length > 1) log.push(textValue); - }); - - return [log, stream]; -}; - export class TechdocsGenerator implements GeneratorBase { private readonly logger: Logger; private readonly containerRunner: ContainerRunner; @@ -74,9 +61,9 @@ export class TechdocsGenerator implements GeneratorBase { outputDir, parsedLocationAnnotation, etag, + logger: childLogger, + logStream, }: GeneratorRunOptions): Promise { - const [log, logStream] = createStream(); - // TODO: In future mkdocs.yml can be mkdocs.yaml. So, use a config variable here to find out // the correct file name. // Do some updates to mkdocs.yml before generating docs e.g. adding repo_url @@ -84,7 +71,7 @@ export class TechdocsGenerator implements GeneratorBase { if (parsedLocationAnnotation) { await patchMkdocsYmlPreBuild( mkdocsYmlPath, - this.logger, + childLogger, parsedLocationAnnotation, ); } @@ -108,7 +95,7 @@ export class TechdocsGenerator implements GeneratorBase { }, logStream, }); - this.logger.info( + childLogger.info( `Successfully generated docs from ${inputDir} into ${outputDir} using local mkdocs`, ); break; @@ -123,7 +110,7 @@ export class TechdocsGenerator implements GeneratorBase { // write to, otherwise they will just fail trying to write to / envVars: { HOME: '/tmp' }, }); - this.logger.info( + childLogger.info( `Successfully generated docs from ${inputDir} into ${outputDir} using techdocs-container`, ); break; @@ -136,7 +123,6 @@ export class TechdocsGenerator implements GeneratorBase { this.logger.debug( `Failed to generate docs from ${inputDir} into ${outputDir}`, ); - this.logger.error(`Build failed with error: ${log}`); throw new Error( `Failed to generate docs from ${inputDir} into ${outputDir} with error ${error.message}`, ); @@ -150,7 +136,7 @@ export class TechdocsGenerator implements GeneratorBase { // Creates techdocs_metadata.json if file does not exist. await addBuildTimestampMetadata( path.join(outputDir, 'techdocs_metadata.json'), - this.logger, + childLogger, ); // Add etag of the prepared tree to techdocs_metadata.json diff --git a/packages/techdocs-common/src/stages/generate/types.ts b/packages/techdocs-common/src/stages/generate/types.ts index 5ea9721c54..185a7175dc 100644 --- a/packages/techdocs-common/src/stages/generate/types.ts +++ b/packages/techdocs-common/src/stages/generate/types.ts @@ -13,8 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Entity } from '@backstage/catalog-model'; import { Writable } from 'stream'; +import { Logger } from 'winston'; import { ParsedLocationAnnotation } from '../../helpers'; /** @@ -24,13 +26,15 @@ import { ParsedLocationAnnotation } from '../../helpers'; * @param {string} outputDir Directory to store generated docs in. Usually - a newly created temporary directory. * @param {ParsedLocationAnnotation} parsedLocationAnnotation backstage.io/techdocs-ref annotation of an entity * @param {string} etag A unique identifier for the prepared tree e.g. commit SHA. If provided it will be stored in techdocs_metadata.json. - * @param {Writable} [logStream] A dedicated log stream + * @param {Logger} [logger] A logger that forwards the messages to the caller to be displayed outside of the backend. + * @param {Writable} [logStream] A log stream that can send raw log messages to the caller to be displayed outside of the backend.. */ export type GeneratorRunOptions = { inputDir: string; outputDir: string; parsedLocationAnnotation?: ParsedLocationAnnotation; etag?: string; + logger: Logger; logStream?: Writable; }; diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 714f3ad442..868e57bbfd 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -31,6 +31,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.8.5", + "@backstage/catalog-client": "^0.3.16", "@backstage/catalog-model": "^0.9.0", "@backstage/config": "^0.1.5", "@backstage/errors": "^0.1.1", @@ -47,6 +48,7 @@ "devDependencies": { "@backstage/cli": "^0.7.3", "@types/dockerode": "^3.2.1", + "msw": "^0.29.0", "supertest": "^6.1.3" }, "files": [ diff --git a/plugins/techdocs-backend/src/DocsBuilder/builder.ts b/plugins/techdocs-backend/src/DocsBuilder/builder.ts index e9df0b8400..2cd6f57154 100644 --- a/plugins/techdocs-backend/src/DocsBuilder/builder.ts +++ b/plugins/techdocs-backend/src/DocsBuilder/builder.ts @@ -18,6 +18,7 @@ import { ENTITY_DEFAULT_NAMESPACE, stringifyEntityRef, } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; import { NotModifiedError } from '@backstage/errors'; import { GeneratorBase, @@ -31,8 +32,8 @@ import { import fs from 'fs-extra'; import os from 'os'; import path from 'path'; +import { Writable } from 'stream'; import { Logger } from 'winston'; -import { Config } from '@backstage/config'; import { BuildMetadataStorage } from './BuildMetadataStorage'; type DocsBuilderArguments = { @@ -42,6 +43,7 @@ type DocsBuilderArguments = { entity: Entity; logger: Logger; config: Config; + logStream?: Writable; }; export class DocsBuilder { @@ -51,6 +53,7 @@ export class DocsBuilder { private entity: Entity; private logger: Logger; private config: Config; + private logStream: Writable | undefined; constructor({ preparers, @@ -59,6 +62,7 @@ export class DocsBuilder { entity, logger, config, + logStream, }: DocsBuilderArguments) { this.preparer = preparers.get(entity); this.generator = generators.get(entity); @@ -66,6 +70,7 @@ export class DocsBuilder { this.entity = entity; this.logger = logger; this.config = config; + this.logStream = logStream; } /** @@ -115,6 +120,7 @@ export class DocsBuilder { try { const preparerResponse = await this.preparer.prepare(this.entity, { etag: storedEtag, + logger: this.logger, }); preparedDir = preparerResponse.preparedDir; @@ -159,12 +165,15 @@ export class DocsBuilder { const outputDir = await fs.mkdtemp( path.join(tmpdirResolvedPath, 'techdocs-tmp-'), ); + const parsedLocationAnnotation = getLocationForEntity(this.entity); await this.generator.run({ inputDir: preparedDir, outputDir, parsedLocationAnnotation, etag: newEtag, + logger: this.logger, + logStream: this.logStream, }); // Remove Prepared directory since it is no longer needed. diff --git a/plugins/techdocs-backend/src/index.ts b/plugins/techdocs-backend/src/index.ts index afbf04aba9..6d1e551629 100644 --- a/plugins/techdocs-backend/src/index.ts +++ b/plugins/techdocs-backend/src/index.ts @@ -14,5 +14,5 @@ * limitations under the License. */ -export * from './service/router'; +export { createRouter } from './service/router'; export * from '@backstage/techdocs-common'; diff --git a/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts b/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts new file mode 100644 index 0000000000..409dc6e128 --- /dev/null +++ b/plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts @@ -0,0 +1,194 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + getVoidLogger, + PluginEndpointDiscovery, +} from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { + GeneratorBuilder, + PreparerBuilder, + PublisherBase, +} from '@backstage/techdocs-common'; +import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder'; +import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer'; + +jest.mock('../DocsBuilder'); + +const MockedDocsBuilder = DocsBuilder as jest.MockedClass; + +describe('DocsSynchronizer', () => { + const preparers: jest.Mocked = { + register: jest.fn(), + get: jest.fn(), + }; + const generators: jest.Mocked = { + register: jest.fn(), + get: jest.fn(), + }; + const publisher: jest.Mocked = { + docsRouter: jest.fn(), + fetchTechDocsMetadata: jest.fn(), + getReadiness: jest.fn(), + hasDocsBeenGenerated: jest.fn(), + publish: jest.fn(), + }; + const discovery: jest.Mocked = { + getBaseUrl: jest.fn(), + getExternalBaseUrl: jest.fn(), + }; + + let docsSynchronizer: DocsSynchronizer; + const mockResponseHandler: jest.Mocked = { + log: jest.fn(), + finish: jest.fn(), + error: jest.fn(), + }; + + beforeEach(async () => { + publisher.docsRouter.mockReturnValue(() => {}); + discovery.getBaseUrl.mockImplementation(async type => { + return `http://backstage.local/api/${type}`; + }); + + docsSynchronizer = new DocsSynchronizer({ + publisher, + config: new ConfigReader({}), + logger: getVoidLogger(), + }); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('doSync', () => { + it('should execute an update', async () => { + (shouldCheckForUpdate as jest.Mock).mockReturnValue(true); + + const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + uid: '0', + name: 'test', + namespace: 'default', + }, + }; + + MockedDocsBuilder.prototype.build.mockImplementation(async () => { + // extract the logStream from the constructor call + const logStream = MockedDocsBuilder.mock.calls[0][0].logStream; + + logStream?.write('Some log'); + logStream?.write('Another log'); + + const logger = MockedDocsBuilder.mock.calls[0][0].logger; + + logger.info('Some more log'); + + return true; + }); + + publisher.hasDocsBeenGenerated.mockResolvedValue(true); + + await docsSynchronizer.doSync({ + responseHandler: mockResponseHandler, + entity, + preparers, + generators, + }); + + expect(mockResponseHandler.log).toBeCalledTimes(3); + expect(mockResponseHandler.log).toBeCalledWith('Some log'); + expect(mockResponseHandler.log).toBeCalledWith('Another log'); + expect(mockResponseHandler.log).toBeCalledWith( + expect.stringMatching(/info.*Some more log/), + ); + + expect(mockResponseHandler.finish).toBeCalledWith({ updated: true }); + + expect(mockResponseHandler.error).toBeCalledTimes(0); + + expect(shouldCheckForUpdate).toBeCalledTimes(1); + expect(DocsBuilder.prototype.build).toBeCalledTimes(1); + }); + + it('should not check for an update too often', async () => { + (shouldCheckForUpdate as jest.Mock).mockReturnValue(false); + + const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + uid: '0', + name: 'test', + namespace: 'default', + }, + }; + + await docsSynchronizer.doSync({ + responseHandler: mockResponseHandler, + entity, + preparers, + generators, + }); + + expect(mockResponseHandler.finish).toBeCalledWith({ updated: false }); + + expect(mockResponseHandler.log).toBeCalledTimes(0); + expect(mockResponseHandler.error).toBeCalledTimes(0); + + expect(shouldCheckForUpdate).toBeCalledTimes(1); + expect(DocsBuilder.prototype.build).toBeCalledTimes(0); + }); + + it('should forward build errors', async () => { + (shouldCheckForUpdate as jest.Mock).mockReturnValue(true); + + const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + uid: '0', + name: 'test', + namespace: 'default', + }, + }; + + const error = new Error('Some random error'); + MockedDocsBuilder.prototype.build.mockRejectedValue(error); + + await docsSynchronizer.doSync({ + responseHandler: mockResponseHandler, + entity, + preparers, + generators, + }); + + expect(mockResponseHandler.log).toBeCalledTimes(1); + expect(mockResponseHandler.log).toBeCalledWith( + expect.stringMatching( + /error.*: Failed to build the docs page: Some random error/, + ), + ); + expect(mockResponseHandler.finish).toBeCalledTimes(0); + expect(mockResponseHandler.error).toBeCalledTimes(1); + expect(mockResponseHandler.error).toBeCalledWith(error); + }); + }); +}); diff --git a/plugins/techdocs-backend/src/service/DocsSynchronizer.ts b/plugins/techdocs-backend/src/service/DocsSynchronizer.ts new file mode 100644 index 0000000000..9b51381748 --- /dev/null +++ b/plugins/techdocs-backend/src/service/DocsSynchronizer.ts @@ -0,0 +1,141 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { NotFoundError } from '@backstage/errors'; +import { + GeneratorBuilder, + PreparerBuilder, + PublisherBase, +} from '@backstage/techdocs-common'; +import { PassThrough } from 'stream'; +import * as winston from 'winston'; +import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder'; + +export type DocsSynchronizerSyncOpts = { + log: (message: string) => void; + error: (e: Error) => void; + finish: (result: { updated: boolean }) => void; +}; + +export class DocsSynchronizer { + private readonly publisher: PublisherBase; + private readonly logger: winston.Logger; + private readonly config: Config; + + constructor({ + publisher, + logger, + config, + }: { + publisher: PublisherBase; + logger: winston.Logger; + config: Config; + }) { + this.config = config; + this.logger = logger; + this.publisher = publisher; + } + + async doSync({ + responseHandler: { log, error, finish }, + entity, + preparers, + generators, + }: { + responseHandler: DocsSynchronizerSyncOpts; + entity: Entity; + preparers: PreparerBuilder; + generators: GeneratorBuilder; + }) { + // create a new logger to log data to the caller + const taskLogger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.colorize(), + winston.format.timestamp(), + winston.format.simple(), + ), + defaultMeta: {}, + }); + + // create an in-memory stream to forward logs to the event-stream + const logStream = new PassThrough(); + logStream.on('data', async data => { + log(data.toString().trim()); + }); + + taskLogger.add(new winston.transports.Stream({ stream: logStream })); + + // check if the last update check was too recent + if (!shouldCheckForUpdate(entity.metadata.uid!)) { + finish({ updated: false }); + return; + } + + const docsBuilder = new DocsBuilder({ + preparers, + generators, + publisher: this.publisher, + logger: taskLogger, + entity, + config: this.config, + logStream, + }); + + let foundDocs = false; + + try { + const updated = await docsBuilder.build(); + + if (!updated) { + finish({ updated: false }); + return; + } + } catch (e) { + const msg = `Failed to build the docs page: ${e.message}`; + taskLogger.error(msg); + this.logger.error(msg, e); + error(e); + return; + } + + // With a maximum of ~5 seconds wait, check if the files got published and if docs will be fetched + // on the user's page. If not, respond with a message asking them to check back later. + // The delay here is to make sure GCS/AWS/etc. registers newly uploaded files which is usually <1 second + for (let attempt = 0; attempt < 5; attempt++) { + if (await this.publisher.hasDocsBeenGenerated(entity)) { + foundDocs = true; + break; + } + await new Promise(r => setTimeout(r, 1000)); + } + if (!foundDocs) { + this.logger.error( + 'Published files are taking longer to show up in storage. Something went wrong.', + ); + error( + new NotFoundError( + 'Sorry! It took too long for the generated docs to show up in storage. Check back later.', + ), + ); + return; + } + + finish({ updated: true }); + } +} diff --git a/plugins/techdocs-backend/src/service/router.test.ts b/plugins/techdocs-backend/src/service/router.test.ts new file mode 100644 index 0000000000..84ab0c6d88 --- /dev/null +++ b/plugins/techdocs-backend/src/service/router.test.ts @@ -0,0 +1,469 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + errorHandler, + getVoidLogger, + PluginEndpointDiscovery, +} from '@backstage/backend-common'; +import { CatalogClient } from '@backstage/catalog-client'; +import { ConfigReader } from '@backstage/config'; +import { NotModifiedError } from '@backstage/errors'; +import { + GeneratorBuilder, + PreparerBuilder, + PublisherBase, +} from '@backstage/techdocs-common'; +import express, { Response } from 'express'; +import request from 'supertest'; +import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer'; +import { createEventStream, createHttpResponse, createRouter } from './router'; + +jest.mock('@backstage/catalog-client'); +jest.mock('@backstage/config'); +jest.mock('./DocsSynchronizer'); + +const MockedConfigReader = ConfigReader as jest.MockedClass< + typeof ConfigReader +>; +const MockCatalogClient = CatalogClient as jest.MockedClass< + typeof CatalogClient +>; +const MockDocsSynchronizer = DocsSynchronizer as jest.MockedClass< + typeof DocsSynchronizer +>; + +describe('createRouter', () => { + const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + uid: '0', + name: 'test', + }, + }; + const entityWithoutMetadata = { + ...entity, + metadata: { + ...entity.metadata, + uid: undefined, + }, + }; + + const preparers: jest.Mocked = { + register: jest.fn(), + get: jest.fn(), + }; + const generators: jest.Mocked = { + register: jest.fn(), + get: jest.fn(), + }; + const publisher: jest.Mocked = { + docsRouter: jest.fn(), + fetchTechDocsMetadata: jest.fn(), + getReadiness: jest.fn(), + hasDocsBeenGenerated: jest.fn(), + publish: jest.fn(), + }; + const discovery: jest.Mocked = { + getBaseUrl: jest.fn(), + getExternalBaseUrl: jest.fn(), + }; + + let app: express.Express; + + beforeEach(() => { + jest.resetAllMocks(); + }); + + beforeEach(async () => { + publisher.docsRouter.mockReturnValue(() => {}); + discovery.getBaseUrl.mockImplementation(async type => { + return `http://backstage.local/api/${type}`; + }); + + const outOfTheBoxRouter = await createRouter({ + preparers, + generators, + publisher, + config: new ConfigReader({}), + logger: getVoidLogger(), + discovery, + }); + const recommendedRouter = await createRouter({ + publisher, + config: new ConfigReader({}), + logger: getVoidLogger(), + discovery, + }); + + app = express(); + app.use(outOfTheBoxRouter); + app.use('/recommended', recommendedRouter); + app.use(errorHandler()); + }); + + describe('GET /sync/:namespace/:kind/:name', () => { + describe('accept application/json', () => { + it('should return not found if entity is not found', async () => { + MockCatalogClient.prototype.getEntityByName.mockResolvedValue( + undefined, + ); + + const response = await request(app) + .get('/sync/default/Component/test') + .send(); + + expect(response.status).toBe(404); + }); + + it('should return not found if entity has no uid', async () => { + MockCatalogClient.prototype.getEntityByName.mockResolvedValue( + entityWithoutMetadata, + ); + + const response = await request(app) + .get('/sync/default/Component/test') + .send(); + + expect(response.status).toBe(404); + }); + + it('should not check for an update without local builder', async () => { + MockedConfigReader.prototype.getString.mockReturnValue('external'); + MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity); + + const response = await request(app) + .get('/sync/default/Component/test') + .send(); + + expect(response.status).toBe(304); + }); + + it('should error if missing builder', async () => { + MockedConfigReader.prototype.getString.mockReturnValue('local'); + MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity); + + const response = await request(app) + .get('/recommended/sync/default/Component/test') + .send(); + + expect(response.status).toBe(500); + expect(response.text).toMatch( + /Invalid configuration\. 'techdocs\.builder' was set to 'local' but no 'preparer' was provided to the router initialization/, + ); + + expect(MockDocsSynchronizer.prototype.doSync).toBeCalledTimes(0); + }); + + it('should execute synchronization', async () => { + MockedConfigReader.prototype.getString.mockReturnValue('local'); + MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity); + MockDocsSynchronizer.prototype.doSync.mockImplementation( + async ({ responseHandler }) => + responseHandler.finish({ updated: true }), + ); + + await request(app).get('/sync/default/Component/test').send(); + + expect(MockDocsSynchronizer.prototype.doSync).toBeCalledTimes(1); + expect(MockDocsSynchronizer.prototype.doSync).toBeCalledWith({ + responseHandler: { + log: expect.any(Function), + error: expect.any(Function), + finish: expect.any(Function), + }, + entity, + generators, + preparers, + }); + }); + + it('should return on updated', async () => { + MockedConfigReader.prototype.getString.mockReturnValue('local'); + MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity); + MockDocsSynchronizer.prototype.doSync.mockImplementation( + async ({ responseHandler }) => { + const { log, finish } = responseHandler; + + log('Some log'); + + finish({ updated: true }); + }, + ); + + const response = await request(app) + .get('/sync/default/Component/test') + .send(); + + expect(response.status).toBe(201); + expect(response.get('content-type')).toMatch(/application\/json/); + expect(response.text).toEqual( + '{"message":"Docs updated or did not need updating"}', + ); + }); + }); + + describe('accept text/event-stream', () => { + it('should return not found if entity is not found', async () => { + MockCatalogClient.prototype.getEntityByName.mockResolvedValue( + undefined, + ); + + const response = await request(app) + .get('/sync/default/Component/test') + .set('accept', 'text/event-stream') + .send(); + + expect(response.status).toBe(404); + }); + + it('should return not found if entity has no uid', async () => { + MockCatalogClient.prototype.getEntityByName.mockResolvedValue( + entityWithoutMetadata, + ); + + const response = await request(app) + .get('/sync/default/Component/test') + .set('accept', 'text/event-stream') + .send(); + + expect(response.status).toBe(404); + }); + + it('should not check for an update without local builder', async () => { + MockedConfigReader.prototype.getString.mockReturnValue('external'); + MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity); + + const response = await request(app) + .get('/sync/default/Component/test') + .set('accept', 'text/event-stream') + .send(); + + expect(response.status).toBe(200); + expect(response.get('content-type')).toBe('text/event-stream'); + expect(response.text).toEqual( + `event: finish +data: {"updated":false} + +`, + ); + }); + + it('should error if missing builder', async () => { + MockedConfigReader.prototype.getString.mockReturnValue('local'); + MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity); + + const response = await request(app) + .get('/recommended/sync/default/Component/test') + .set('accept', 'text/event-stream') + .send(); + + expect(response.status).toBe(200); + expect(response.get('content-type')).toBe('text/event-stream'); + expect(response.text).toEqual( + `event: error +data: "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'preparer' was provided to the router initialization." + +`, + ); + + expect(MockDocsSynchronizer.prototype.doSync).toBeCalledTimes(0); + }); + + it('should execute synchronization', async () => { + MockedConfigReader.prototype.getString.mockReturnValue('local'); + MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity); + MockDocsSynchronizer.prototype.doSync.mockImplementation( + async ({ responseHandler }) => + responseHandler.finish({ updated: true }), + ); + + await request(app) + .get('/sync/default/Component/test') + .set('accept', 'text/event-stream') + .send(); + + expect(MockDocsSynchronizer.prototype.doSync).toBeCalledTimes(1); + expect(MockDocsSynchronizer.prototype.doSync).toBeCalledWith({ + responseHandler: { + log: expect.any(Function), + error: expect.any(Function), + finish: expect.any(Function), + }, + entity, + generators, + preparers, + }); + }); + + it('should return an event-stream', async () => { + MockedConfigReader.prototype.getString.mockReturnValue('local'); + MockCatalogClient.prototype.getEntityByName.mockResolvedValue(entity); + MockDocsSynchronizer.prototype.doSync.mockImplementation( + async ({ responseHandler }) => { + const { log, finish } = responseHandler; + + log('Some log'); + log('Another log'); + + finish({ updated: true }); + }, + ); + + const response = await request(app) + .get('/sync/default/Component/test') + .set('accept', 'text/event-stream') + .send(); + + expect(response.status).toBe(200); + expect(response.get('content-type')).toBe('text/event-stream'); + expect(response.text).toEqual( + `event: log +data: "Some log" + +event: log +data: "Another log" + +event: finish +data: {"updated":true} + +`, + ); + }); + }); + }); +}); + +describe('createEventStream', () => { + const res: jest.Mocked = { + writeHead: jest.fn(), + write: jest.fn(), + end: jest.fn(), + } as any; + + let handlers: DocsSynchronizerSyncOpts; + + beforeEach(() => { + handlers = createEventStream(res); + }); + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should return correct event stream', async () => { + // called in beforeEach + + expect(res.writeHead).toBeCalledTimes(1); + expect(res.writeHead).toBeCalledWith(200, { + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + 'Content-Type': 'text/event-stream', + }); + }); + + it('should flush after write if defined', async () => { + res.flush = jest.fn(); + + handlers.log('A Message'); + + expect(res.write).toBeCalledTimes(1); + expect(res.write).toBeCalledWith(`event: log +data: "A Message" + +`); + expect(res.flush).toBeCalledTimes(1); + }); + + it('should write log', async () => { + handlers.log('A Message'); + + expect(res.write).toBeCalledTimes(1); + expect(res.write).toBeCalledWith(`event: log +data: "A Message" + +`); + expect(res.end).toBeCalledTimes(0); + }); + + it('should write error and end the connection', async () => { + handlers.error(new Error('Some Error')); + + expect(res.write).toBeCalledTimes(1); + expect(res.write).toBeCalledWith(`event: error +data: "Some Error" + +`); + expect(res.end).toBeCalledTimes(1); + }); + + it('should finish and end the connection', async () => { + handlers.finish({ updated: true }); + + expect(res.write).toBeCalledTimes(1); + expect(res.write).toBeCalledWith(`event: finish +data: {"updated":true} + +`); + + expect(res.end).toBeCalledTimes(1); + }); +}); + +describe('createHttpResponse', () => { + const res: jest.Mocked = { + status: jest.fn(), + json: jest.fn(), + } as any; + + let handlers: DocsSynchronizerSyncOpts; + + beforeEach(() => { + res.status.mockImplementation(() => res); + handlers = createHttpResponse(res); + }); + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should return CREATED if updated', async () => { + handlers.finish({ updated: true }); + + expect(res.status).toBeCalledTimes(1); + expect(res.status).toBeCalledWith(201); + + expect(res.json).toBeCalledTimes(1); + expect(res.json).toBeCalledWith({ + message: 'Docs updated or did not need updating', + }); + }); + + it('should return NOT_MODIFIED if not updated', async () => { + expect(() => handlers.finish({ updated: false })).toThrowError( + NotModifiedError, + ); + }); + + it('should throw custom error', async () => { + expect(() => handlers.error(new Error('Some Error'))).toThrowError( + /Some Error/, + ); + }); + + it('should ignore logs', async () => { + expect(() => handlers.log('Some Message')).not.toThrow(); + }); +}); diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 7563dfa2f9..b889312abd 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -14,6 +14,7 @@ * limitations under the License. */ import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { CatalogClient } from '@backstage/catalog-client'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { NotFoundError, NotModifiedError } from '@backstage/errors'; @@ -24,12 +25,11 @@ import { PublisherBase, } from '@backstage/techdocs-common'; import fetch from 'cross-fetch'; -import express from 'express'; +import express, { Response } from 'express'; import Router from 'express-promise-router'; import { Knex } from 'knex'; import { Logger } from 'winston'; -import { DocsBuilder } from '../DocsBuilder'; -import { shouldCheckForUpdate } from '../DocsBuilder/BuildMetadataStorage'; +import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer'; /** * All of the required dependencies for running TechDocs in the "out-of-the-box" @@ -78,6 +78,12 @@ export async function createRouter( ): Promise { const router = Router(); const { publisher, config, logger, discovery } = options; + const catalogClient = new CatalogClient({ discoveryApi: discovery }); + const docsSynchronizer = new DocsSynchronizer({ + publisher: publisher, + logger: logger, + config: config, + }); router.get('/metadata/techdocs/:namespace/:kind/:name', async (req, res) => { const { kind, namespace, name } = req.params; @@ -136,114 +142,58 @@ export async function createRouter( }); // Check if docs are the latest version and trigger rebuilds if not - // Responds with immediate success if rebuild not needed + // Responds with an event-stream that closes after the build finished + // Responds with an immediate success if rebuild not needed // If a build is required, responds with a success when finished router.get('/sync/:namespace/:kind/:name', async (req, res) => { const { kind, namespace, name } = req.params; - const catalogUrl = await discovery.getBaseUrl('catalog'); - const triple = [kind, namespace, name].map(encodeURIComponent).join('/'); - const token = getBearerToken(req.headers.authorization); - const catalogRes = await fetch(`${catalogUrl}/entities/by-name/${triple}`, { - headers: token ? { Authorization: `Bearer ${token}` } : {}, - }); - if (!catalogRes.ok) { - const catalogResText = await catalogRes.text(); - res.status(catalogRes.status); - res.send(catalogResText); - return; - } - const entity: Entity = await catalogRes.json(); + const entity = await catalogClient.getEntityByName( + { kind, namespace, name }, + { token }, + ); - if (!entity.metadata.uid) { + if (!entity?.metadata?.uid) { throw new NotFoundError('Entity metadata UID missing'); } - if (!shouldCheckForUpdate(entity.metadata.uid)) { - res.status(200).json({ - message: `Last check for documentation update is recent, did not retry.`, - }); - return; + + let responseHandler: DocsSynchronizerSyncOpts; + if (req.header('accept') !== 'text/event-stream') { + console.warn( + "The call to /sync/:namespace/:kind/:name wasn't done by an EventSource. This behavior is deprecated and will be removed soon. Make sure to update the @backstage/plugin-techdocs package in the frontend to the latest version.", + ); + responseHandler = createHttpResponse(res); + } else { + responseHandler = createEventStream(res); } - let publisherType = ''; - try { - publisherType = config.getString('techdocs.publisher.type'); - } catch (err) { - throw new Error( - 'Unable to get techdocs.publisher.type in your app config. Set it to either ' + - "'local', 'googleGcs' or other support storage providers. Read more here " + - 'https://backstage.io/docs/features/techdocs/architecture', - ); - } // techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to 'local' // If set to 'external', it will assume that an external process (e.g. CI/CD pipeline // of the repository) is responsible for building and publishing documentation to the storage provider if (config.getString('techdocs.builder') !== 'local') { - res.status(200).json({ - message: - '`techdocs.builder` app config is not set to `local`, so docs will not be generated locally and sync is not required.', + responseHandler.finish({ updated: false }); + return; + } + + // Set the synchronization and build process if "out-of-the-box" configuration is provided. + if (isOutOfTheBoxOption(options)) { + const { preparers, generators } = options; + + await docsSynchronizer.doSync({ + responseHandler, + entity, + preparers, + generators, }); return; } - // Set up a DocsBuilder if "out-of-the-box" configuration is provided. - if (isOutOfTheBoxOption(options)) { - const { preparers, generators } = options; - const docsBuilder = new DocsBuilder({ - preparers, - generators, - publisher, - logger, - entity, - config, - }); - let foundDocs = false; - switch (publisherType) { - case 'local': - case 'awsS3': - case 'azureBlobStorage': - case 'openStackSwift': - case 'googleGcs': { - // This block should be valid for all storage implementations. So no need to duplicate in future, - // add the publisher type in the list here. - const updated = await docsBuilder.build(); - - if (!updated) { - throw new NotModifiedError(); - } - - // With a maximum of ~5 seconds wait, check if the files got published and if docs will be fetched - // on the user's page. If not, respond with a message asking them to check back later. - // The delay here is to make sure GCS/AWS/etc. registers newly uploaded files which is usually <1 second - for (let attempt = 0; attempt < 5; attempt++) { - if (await publisher.hasDocsBeenGenerated(entity)) { - foundDocs = true; - break; - } - await new Promise(r => setTimeout(r, 1000)); - } - if (!foundDocs) { - logger.error( - 'Published files are taking longer to show up in storage. Something went wrong.', - ); - throw new NotFoundError( - 'Sorry! It took too long for the generated docs to show up in storage. Check back later.', - ); - } - - res - .status(201) - .json({ message: 'Docs updated or did not need updating' }); - break; - } - - default: - throw new NotFoundError( - `Publisher type ${publisherType} is not supported by techdocs-backend docs builder.`, - ); - } - } + responseHandler.error( + new Error( + "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'preparer' was provided to the router initialization.", + ), + ); }); // Route middleware which serves files from the storage set in the publisher. @@ -255,3 +205,79 @@ export async function createRouter( function getBearerToken(header?: string): string | undefined { return header?.match(/(?:Bearer)\s+(\S+)/i)?.[1]; } + +/** + * Create an event-stream response that emits the events 'log', 'error', and 'finish'. + * + * @param res the response to write the event-stream to + * @returns A tuple of callbacks to emit messages. A call to 'error' or 'finish' + * will close the event-stream. + */ +export function createEventStream( + res: Response, +): DocsSynchronizerSyncOpts { + // Mandatory headers and http status to keep connection open + res.writeHead(200, { + Connection: 'keep-alive', + 'Cache-Control': 'no-cache', + 'Content-Type': 'text/event-stream', + }); + + // client closes connection + res.socket?.on('close', () => { + res.end(); + }); + + // write the event to the stream + const send = (type: 'error' | 'finish' | 'log', data: any) => { + res.write(`event: ${type}\ndata: ${JSON.stringify(data)}\n\n`); + + // res.flush() is only available with the compression middleware + if (res.flush) { + res.flush(); + } + }; + + return { + log: data => { + send('log', data); + }, + + error: e => { + send('error', e.message); + res.end(); + }, + + finish: result => { + send('finish', result); + res.end(); + }, + }; +} + +/** + * Create a HTTP response. This is used for the legacy non-event-stream implementation of the sync endpoint. + * + * @param res the response to write the event-stream to + * @returns A tuple of callbacks to emit messages. A call to 'error' or 'finish' + * will close the event-stream. + */ +export function createHttpResponse( + res: Response, +): DocsSynchronizerSyncOpts { + return { + log: () => {}, + error: e => { + throw e; + }, + finish: ({ updated }) => { + if (!updated) { + throw new NotModifiedError(); + } + + res + .status(201) + .json({ message: 'Docs updated or did not need updating' }); + }, + }; +} diff --git a/plugins/techdocs/api-report.md b/plugins/techdocs/api-report.md index 943831d91c..0f55d3f33d 100644 --- a/plugins/techdocs/api-report.md +++ b/plugins/techdocs/api-report.md @@ -50,7 +50,7 @@ export const Reader: ({ entityId, onReady }: Props_2) => JSX.Element; export const Router: () => JSX.Element; // @public (undocumented) -export type SyncResult = 'cached' | 'updated' | 'timeout'; +export type SyncResult = 'cached' | 'updated'; // @public (undocumented) export interface TechDocsApi { @@ -129,7 +129,10 @@ export interface TechDocsStorageApi { // (undocumented) getStorageUrl(): Promise; // (undocumented) - syncEntityDocs(entityId: EntityName): Promise; + syncEntityDocs( + entityId: EntityName, + logHandler?: (line: string) => void, + ): Promise; } // @public (undocumented) @@ -165,7 +168,10 @@ export class TechDocsStorageClient implements TechDocsStorageApi { getStorageUrl(): Promise; // (undocumented) identityApi: IdentityApi; - syncEntityDocs(entityId: EntityName): Promise; + syncEntityDocs( + entityId: EntityName, + logHandler?: (line: string) => void, + ): Promise; } // (No @packageDocumentation comment for this package) diff --git a/plugins/techdocs/dev/index.tsx b/plugins/techdocs/dev/index.tsx index cdc81caa5a..659a4e2caf 100644 --- a/plugins/techdocs/dev/index.tsx +++ b/plugins/techdocs/dev/index.tsx @@ -17,6 +17,7 @@ import { createDevApp } from '@backstage/dev-utils'; import { NotFoundError } from '@backstage/errors'; import React from 'react'; +import { EntityName } from '@backstage/catalog-model'; import { Reader, SyncResult, @@ -81,9 +82,18 @@ function createPage({ }); } - async syncEntityDocs() { + async syncEntityDocs(_: EntityName, logHandler?: (line: string) => void) { if (syncDocsDelay) { - await new Promise(resolve => setTimeout(resolve, syncDocsDelay)); + for (let i = 0; i < 10; i++) { + setTimeout( + () => logHandler?.call(this, `Log line ${i}`), + ((i + 1) * syncDocsDelay) / 10, + ); + } + + await new Promise(resolve => { + setTimeout(resolve, syncDocsDelay); + }); } return syncDocs(); @@ -138,6 +148,11 @@ createDevApp() {createPage({ + entityDocs: ({ called, content }) => { + return called === 0 + ? content + : content.replace(/World/, 'New World'); + }, syncDocs: () => 'updated', syncDocsDelay: 2000, })} @@ -195,13 +210,6 @@ createDevApp() syncDocsDelay: 2000, })} - - - {createPage({ - syncDocs: () => 'timeout', - syncDocsDelay: 2000, - })} - ), diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index 3b348873aa..4c71084a8a 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -44,8 +44,10 @@ "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", "@material-ui/styles": "^4.10.0", + "eventsource": "^1.1.0", "react": "^16.13.1", "react-dom": "^16.13.1", + "react-lazylog": "^4.5.2", "react-router": "6.0.0-beta.0", "react-router-dom": "6.0.0-beta.0", "react-use": "^17.2.4", @@ -60,6 +62,7 @@ "@testing-library/react": "^11.2.5", "@testing-library/react-hooks": "^3.4.2", "@testing-library/user-event": "^13.1.8", + "@types/eventsource": "^1.1.5", "@types/jest": "^26.0.7", "@types/node": "^14.14.32", "@types/react": "^16.9", diff --git a/plugins/techdocs/src/api.ts b/plugins/techdocs/src/api.ts index 739c5fc133..a23d9fc7f0 100644 --- a/plugins/techdocs/src/api.ts +++ b/plugins/techdocs/src/api.ts @@ -28,14 +28,17 @@ export const techdocsApiRef = createApiRef({ description: 'Used to make requests towards techdocs API', }); -export type SyncResult = 'cached' | 'updated' | 'timeout'; +export type SyncResult = 'cached' | 'updated'; export interface TechDocsStorageApi { getApiOrigin(): Promise; getStorageUrl(): Promise; getBuilder(): Promise; getEntityDocs(entityId: EntityName, path: string): Promise; - syncEntityDocs(entityId: EntityName): Promise; + syncEntityDocs( + entityId: EntityName, + logHandler?: (line: string) => void, + ): Promise; getBaseUrl( oldBaseUrl: string, entityId: EntityName, diff --git a/plugins/techdocs/src/client.test.ts b/plugins/techdocs/src/client.test.ts index 329d3cbcb6..982c9496b2 100644 --- a/plugins/techdocs/src/client.test.ts +++ b/plugins/techdocs/src/client.test.ts @@ -13,9 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Config } from '@backstage/config'; -import { TechDocsStorageClient } from './client'; import { UrlPatternDiscovery } from '@backstage/core-app-api'; +import { IdentityApi } from '@backstage/core-plugin-api'; +import { NotFoundError } from '@backstage/errors'; +import EventSource from 'eventsource'; +import { TechDocsStorageClient } from './client'; + +const MockedEventSource: jest.MockedClass< + typeof EventSource +> = EventSource as any; + +jest.mock('eventsource'); const mockEntity = { kind: 'Component', @@ -29,6 +39,16 @@ describe('TechDocsStorageClient', () => { getOptionalString: () => 'http://backstage:9191/api/techdocs', } as Partial; const discoveryApi = UrlPatternDiscovery.compile(mockBaseUrl); + const identityApi: jest.Mocked = { + getIdToken: jest.fn(), + getProfile: jest.fn(), + getUserId: jest.fn(), + signOut: jest.fn(), + }; + + beforeEach(() => { + jest.resetAllMocks(); + }); it('should return correct base url based on defined storage', async () => { // @ts-ignore Partial not assignable to Config. @@ -51,4 +71,183 @@ describe('TechDocsStorageClient', () => { `${mockBaseUrl}/static/docs/${mockEntity.namespace}/${mockEntity.kind}/${mockEntity.name}/test/`, ); }); + + describe('syncEntityDocs', () => { + it('should create eventsource without headers', async () => { + const storageApi = new TechDocsStorageClient({ + // @ts-ignore Partial not assignable to Config. + configApi, + discoveryApi, + identityApi, + }); + + MockedEventSource.prototype.addEventListener.mockImplementation( + (type, fn) => { + if (type === 'finish') { + fn({ data: '{"updated": false}' } as any); + } + }, + ); + + await storageApi.syncEntityDocs(mockEntity); + + expect( + MockedEventSource, + ).toBeCalledWith( + 'http://backstage:9191/api/techdocs/sync/default/Component/test-component', + { withCredentials: true, headers: {} }, + ); + }); + + it('should create eventsource with headers', async () => { + const storageApi = new TechDocsStorageClient({ + // @ts-ignore Partial not assignable to Config. + configApi, + discoveryApi, + identityApi, + }); + + MockedEventSource.prototype.addEventListener.mockImplementation( + (type, fn) => { + if (type === 'finish') { + fn({ data: '{"updated": false}' } as any); + } + }, + ); + + identityApi.getIdToken.mockResolvedValue('token'); + + await storageApi.syncEntityDocs(mockEntity); + + expect( + MockedEventSource, + ).toBeCalledWith( + 'http://backstage:9191/api/techdocs/sync/default/Component/test-component', + { withCredentials: true, headers: { Authorization: 'Bearer token' } }, + ); + }); + + it('should resolve to cached', async () => { + const storageApi = new TechDocsStorageClient({ + // @ts-ignore Partial not assignable to Config. + configApi, + discoveryApi, + identityApi, + }); + + MockedEventSource.prototype.addEventListener.mockImplementation( + (type, fn) => { + if (type === 'finish') { + fn({ data: '{"updated": false}' } as any); + } + }, + ); + + await expect(storageApi.syncEntityDocs(mockEntity)).resolves.toEqual( + 'cached', + ); + }); + + it('should resolve to updated', async () => { + const storageApi = new TechDocsStorageClient({ + // @ts-ignore Partial not assignable to Config. + configApi, + discoveryApi, + identityApi, + }); + + MockedEventSource.prototype.addEventListener.mockImplementation( + (type, fn) => { + if (type === 'finish') { + fn({ data: '{"updated": true}' } as any); + } + }, + ); + + await expect(storageApi.syncEntityDocs(mockEntity)).resolves.toEqual( + 'updated', + ); + }); + + it('should log values', async () => { + const storageApi = new TechDocsStorageClient({ + // @ts-ignore Partial not assignable to Config. + configApi, + discoveryApi, + identityApi, + }); + + MockedEventSource.prototype.addEventListener.mockImplementation( + (type, fn) => { + if (type === 'log') { + fn({ data: '"A log message"' } as any); + } + + if (type === 'finish') { + fn({ data: '{"updated": false}' } as any); + } + }, + ); + + const logHandler = jest.fn(); + await expect( + storageApi.syncEntityDocs(mockEntity, logHandler), + ).resolves.toEqual('cached'); + + expect(logHandler).toBeCalledTimes(1); + expect(logHandler).toBeCalledWith('A log message'); + }); + + it('should throw NotFoundError', async () => { + const storageApi = new TechDocsStorageClient({ + // @ts-ignore Partial not assignable to Config. + configApi, + discoveryApi, + identityApi, + }); + + // we await later after we emitted the error + const promise = storageApi.syncEntityDocs(mockEntity).then(); + + // flush the event loop + await new Promise(setImmediate); + + const instance = MockedEventSource.mock + .instances[0] as jest.Mocked; + + instance.onerror({ + status: 404, + message: 'Some not found warning', + } as any); + + await expect(promise).rejects.toThrow(NotFoundError); + await expect(promise).rejects.toThrowError('Some not found warning'); + }); + + it('should throw generic errors', async () => { + const storageApi = new TechDocsStorageClient({ + // @ts-ignore Partial not assignable to Config. + configApi, + discoveryApi, + identityApi, + }); + + // we await later after we emitted the error + const promise = storageApi.syncEntityDocs(mockEntity).then(); + + // flush the event loop + await new Promise(setImmediate); + + const instance = MockedEventSource.mock + .instances[0] as jest.Mocked; + + instance.onerror({ + type: 'error', + data: 'Some other error', + } as any); + + await expect(promise).rejects.toThrow(Error); + await expect(promise).rejects.toThrowError('Some other error'); + }); + }); }); diff --git a/plugins/techdocs/src/client.ts b/plugins/techdocs/src/client.ts index 19fc565a2f..afbf4b74a0 100644 --- a/plugins/techdocs/src/client.ts +++ b/plugins/techdocs/src/client.ts @@ -16,10 +16,11 @@ import { EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; +import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api'; import { NotFoundError } from '@backstage/errors'; +import EventSource from 'eventsource'; import { SyncResult, TechDocsApi, TechDocsStorageApi } from './api'; import { TechDocsEntityMetadata, TechDocsMetadata } from './types'; -import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api'; /** * API to talk to techdocs-backend. @@ -192,44 +193,59 @@ export class TechDocsStorageClient implements TechDocsStorageApi { * Check if docs are on the latest version and trigger rebuild if not * * @param {EntityName} entityId Object containing entity data like name, namespace, etc. + * @param {Function} logHandler Callback to receive log messages from the build process * @returns {SyncResult} Whether documents are currently synchronized to newest version * @throws {Error} Throws error on error from sync endpoint in Techdocs Backend */ - async syncEntityDocs(entityId: EntityName): Promise { + async syncEntityDocs( + entityId: EntityName, + logHandler: (line: string) => void = () => {}, + ): Promise { const { kind, namespace, name } = entityId; const apiOrigin = await this.getApiOrigin(); const url = `${apiOrigin}/sync/${namespace}/${kind}/${name}`; const token = await this.identityApi.getIdToken(); - let request; - let attempts: number = 0; - // retry if request times out, up to 5 times - // can happen due to docs taking too long to generate - while (!request || (request.status === 408 && attempts < 5)) { - attempts++; - request = await fetch(url, { + + return new Promise((resolve, reject) => { + const source = new EventSource(url, { + withCredentials: true, headers: token ? { Authorization: `Bearer ${token}` } : {}, }); - } - switch (request.status) { - case 404: - throw new NotFoundError((await request.json()).error); + source.addEventListener('log', (e: any) => { + if (e.data) { + logHandler(JSON.parse(e.data)); + } + }); - case 200: - case 304: - return 'cached'; + source.addEventListener('finish', (e: any) => { + let updated: boolean = false; - case 201: - return 'updated'; + if (e.data) { + ({ updated } = JSON.parse(e.data)); + } - // for timeout and misc errors, handle without error to allow viewing older docs - // if older docs not available, - // Reader will show 404 error coming from getEntityDocs - case 408: - default: - return 'timeout'; - } + resolve(updated ? 'updated' : 'cached'); + }); + + source.onerror = (e: any) => { + source.close(); + + switch (e.status) { + // the endpoint returned a 404 status + case 404: + reject(new NotFoundError(e.message)); + return; + + // also handles the event-stream close. the reject is ignored if the Promise was already + // resolved by a finish event. + default: + reject(new Error(e.data)); + return; + } + }; + }); } async getBaseUrl( diff --git a/plugins/techdocs/src/reader/components/Reader.tsx b/plugins/techdocs/src/reader/components/Reader.tsx index 3bf72e6004..d63ddf8125 100644 --- a/plugins/techdocs/src/reader/components/Reader.tsx +++ b/plugins/techdocs/src/reader/components/Reader.tsx @@ -15,9 +15,11 @@ */ import { EntityName } from '@backstage/catalog-model'; +import { Progress } from '@backstage/core-components'; +import { useApi } from '@backstage/core-plugin-api'; import { scmIntegrationsApiRef } from '@backstage/integration-react'; import { BackstageTheme } from '@backstage/theme'; -import { useTheme } from '@material-ui/core'; +import { Button, CircularProgress, useTheme } from '@material-ui/core'; import { Alert } from '@material-ui/lab'; import React, { useCallback, useEffect, useRef, useState } from 'react'; import { useNavigate, useParams } from 'react-router-dom'; @@ -34,10 +36,9 @@ import { simplifyMkdocsFooter, transform as transformer, } from '../transformers'; +import { TechDocsBuildLogs } from './TechDocsBuildLogs'; import { TechDocsNotFound } from './TechDocsNotFound'; -import TechDocsProgressBar from './TechDocsProgressBar'; import { useReaderState } from './useReaderState'; -import { useApi } from '@backstage/core-plugin-api'; type Props = { entityId: EntityName; @@ -49,12 +50,14 @@ export const Reader = ({ entityId, onReady }: Props) => { const { '*': path } = useParams(); const theme = useTheme(); - const { state, content: rawPage, errorMessage } = useReaderState( - kind, - namespace, - name, - path, - ); + const { + state, + contentReload, + content: rawPage, + contentErrorMessage, + syncErrorMessage, + buildLog, + } = useReaderState(kind, namespace, name, path); const techdocsStorageApi = useApi(techdocsStorageApiRef); const [sidebars, setSidebars] = useState(); @@ -324,34 +327,67 @@ export const Reader = ({ entityId, onReady }: Props) => { return ( <> - {(state === 'CHECKING' || state === 'INITIAL_BUILD') && ( - + {state === 'CHECKING' && } + {state === 'INITIAL_BUILD' && ( + } + action={} + > + Documentation is accessed for the first time and is being prepared. + The subsequent loads are much faster. + )} {state === 'CONTENT_STALE_REFRESHING' && ( - + } + action={} + > A newer version of this documentation is being prepared and will be available shortly. )} {state === 'CONTENT_STALE_READY' && ( - + contentReload()}> + Refresh + + } + > A newer version of this documentation is now available, please refresh to view. )} - {state === 'CONTENT_STALE_TIMEOUT' && ( - - Building a newer version of this documentation took longer than - expected. Please refresh to try again. - - )} {state === 'CONTENT_STALE_ERROR' && ( - - Building a newer version of this documentation failed. {errorMessage} + } + > + Building a newer version of this documentation failed.{' '} + {syncErrorMessage} )} {state === 'CONTENT_NOT_FOUND' && ( - + <> + {syncErrorMessage && ( + } + > + Building a newer version of this documentation failed.{' '} + {syncErrorMessage} + + )} + + )}
diff --git a/plugins/techdocs/src/reader/components/TechDocsBuildLogs.test.tsx b/plugins/techdocs/src/reader/components/TechDocsBuildLogs.test.tsx new file mode 100644 index 0000000000..d1bb99b76c --- /dev/null +++ b/plugins/techdocs/src/reader/components/TechDocsBuildLogs.test.tsx @@ -0,0 +1,87 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { render } from '@testing-library/react'; +import React from 'react'; +import { + TechDocsBuildLogs, + TechDocsBuildLogsDrawerContent, +} from './TechDocsBuildLogs'; + +// react-lazylog is based on a react-virtualized component which doesn't +// write the content to the dom, so we mock it. +jest.mock('react-lazylog', () => { + return { + LazyLog: ({ text }: { text: string }) => { + return

{text}

; + }, + }; +}); + +describe('', () => { + it('should render with button', () => { + const rendered = render(); + expect(rendered.getByText(/Show Build Logs/i)).toBeInTheDocument(); + expect(rendered.queryByText(/Build Details/i)).not.toBeInTheDocument(); + }); + + it('should open drawer', () => { + const rendered = render(); + rendered.getByText(/Show Build Logs/i).click(); + expect(rendered.getByText(/Build Details/i)).toBeInTheDocument(); + }); +}); + +describe('', () => { + it('should render with empty log', () => { + const onClose = jest.fn(); + const rendered = render( + , + ); + expect(rendered.getByText(/Build Details/i)).toBeInTheDocument(); + expect(rendered.getByText(/Waiting for logs.../i)).toBeInTheDocument(); + + expect(onClose).toBeCalledTimes(0); + }); + + it('should render with empty logs', () => { + const onClose = jest.fn(); + const rendered = render( + , + ); + expect(rendered.getByText(/Build Details/i)).toBeInTheDocument(); + expect( + rendered.queryByText(/Waiting for logs.../i), + ).not.toBeInTheDocument(); + expect(rendered.getByText(/Line 1/i)).toBeInTheDocument(); + expect(rendered.getByText(/Line 2/i)).toBeInTheDocument(); + + expect(onClose).toBeCalledTimes(0); + }); + + it('should call onClose', () => { + const onClose = jest.fn(); + const rendered = render( + , + ); + rendered.getByRole('button').click(); + + expect(onClose).toBeCalledTimes(1); + }); +}); diff --git a/plugins/techdocs/src/reader/components/TechDocsBuildLogs.tsx b/plugins/techdocs/src/reader/components/TechDocsBuildLogs.tsx new file mode 100644 index 0000000000..d9da954152 --- /dev/null +++ b/plugins/techdocs/src/reader/components/TechDocsBuildLogs.tsx @@ -0,0 +1,121 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Button, + createStyles, + Drawer, + Grid, + IconButton, + makeStyles, + Theme, + Typography, +} from '@material-ui/core'; +import Close from '@material-ui/icons/Close'; +import * as React from 'react'; +import { useState } from 'react'; +import { LazyLog } from 'react-lazylog'; + +const useDrawerStyles = makeStyles((theme: Theme) => + createStyles({ + paper: { + width: '100%', + [theme.breakpoints.up('sm')]: { + width: '75%', + }, + [theme.breakpoints.up('md')]: { + width: '50%', + }, + padding: theme.spacing(2.5), + }, + root: { + height: '100%', + overflow: 'hidden', + }, + }), +); + +export const TechDocsBuildLogsDrawerContent = ({ + buildLog, + onClose, +}: { + buildLog: string[]; + onClose: () => void; +}) => { + const classes = useDrawerStyles(); + return ( + + + Build Details + + + + + + + + ); +}; + +export const TechDocsBuildLogs = ({ buildLog }: { buildLog: string[] }) => { + const classes = useDrawerStyles(); + const [open, setOpen] = useState(false); + + return ( + <> + + setOpen(false)} + > + setOpen(false)} + /> + + + ); +}; diff --git a/plugins/techdocs/src/reader/components/TechDocsProgressBar.test.tsx b/plugins/techdocs/src/reader/components/TechDocsProgressBar.test.tsx deleted file mode 100644 index f33ed62b97..0000000000 --- a/plugins/techdocs/src/reader/components/TechDocsProgressBar.test.tsx +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import TechDocsProgressBar from './TechDocsProgressBar'; -import React from 'react'; -import { render } from '@testing-library/react'; -import { wrapInTestApp } from '@backstage/test-utils'; -import { act } from 'react-dom/test-utils'; - -jest.useFakeTimers(); - -describe('', () => { - it('should render a message if techdocs page takes more time to load', () => { - const rendered = render(wrapInTestApp()); - act(() => { - jest.advanceTimersByTime(250); - }); - expect(rendered.getByTestId('progress')).toBeInTheDocument(); - expect(rendered.queryByTestId('delay-reason')).toBeNull(); - act(() => { - jest.advanceTimersByTime(5000); - }); - expect(rendered.getByTestId('delay-reason')).toBeInTheDocument(); - }); -}); diff --git a/plugins/techdocs/src/reader/components/TechDocsProgressBar.tsx b/plugins/techdocs/src/reader/components/TechDocsProgressBar.tsx deleted file mode 100644 index 7089f5ae08..0000000000 --- a/plugins/techdocs/src/reader/components/TechDocsProgressBar.tsx +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import React, { useState, useEffect } from 'react'; -import { useMountedState } from 'react-use'; -import { Typography } from '@material-ui/core'; -import { Progress } from '@backstage/core-components'; - -const TechDocsProgressBar = () => { - const isMounted = useMountedState(); - const [hasBeenDelayed, setHasBeenDelayed] = useState(false); - - const delayReason = `Docs are still loading...Backstage takes some extra time to load docs - for the first time. The subsequent loads are much faster.`; - - // Allowed time that docs can take to load (in milliseconds) - const allowedDelayTime = 5000; - - useEffect(() => { - setTimeout(() => { - if (isMounted()) { - setHasBeenDelayed(true); - } - }, allowedDelayTime); - }); - - return ( - <> - {hasBeenDelayed ? ( - {delayReason} - ) : null} - - - ); -}; - -export default TechDocsProgressBar; diff --git a/plugins/techdocs/src/reader/components/useReaderState.test.tsx b/plugins/techdocs/src/reader/components/useReaderState.test.tsx index 1edf41af30..9d9a8e2b1a 100644 --- a/plugins/techdocs/src/reader/components/useReaderState.test.tsx +++ b/plugins/techdocs/src/reader/components/useReaderState.test.tsx @@ -14,6 +14,7 @@ * limitations under the License. */ +import { ApiProvider, ApiRegistry } from '@backstage/core-app-api'; import { NotFoundError } from '@backstage/errors'; import { act, renderHook } from '@testing-library/react-hooks'; import React from 'react'; @@ -23,7 +24,6 @@ import { reducer, useReaderState, } from './useReaderState'; -import { ApiProvider, ApiRegistry } from '@backstage/core-app-api'; describe('useReaderState', () => { let Wrapper: React.ComponentType; @@ -55,14 +55,12 @@ describe('useReaderState', () => { ${false} | ${undefined} | ${'BUILDING'} | ${'INITIAL_BUILD'} ${false} | ${undefined} | ${'BUILD_READY'} | ${'CONTENT_NOT_FOUND'} ${false} | ${undefined} | ${'BUILD_READY_RELOAD'} | ${'CHECKING'} - ${false} | ${undefined} | ${'BUILD_TIMED_OUT'} | ${'CONTENT_NOT_FOUND'} ${false} | ${undefined} | ${'UP_TO_DATE'} | ${'CONTENT_NOT_FOUND'} ${false} | ${undefined} | ${'ERROR'} | ${'CONTENT_NOT_FOUND'} ${false} | ${'asdf'} | ${'CHECKING'} | ${'CONTENT_FRESH'} ${false} | ${'asdf'} | ${'BUILDING'} | ${'CONTENT_STALE_REFRESHING'} ${false} | ${'asdf'} | ${'BUILD_READY'} | ${'CONTENT_STALE_READY'} ${false} | ${'asdf'} | ${'BUILD_READY_RELOAD'} | ${'CHECKING'} - ${false} | ${'asdf'} | ${'BUILD_TIMED_OUT'} | ${'CONTENT_STALE_TIMEOUT'} ${false} | ${'asdf'} | ${'UP_TO_DATE'} | ${'CONTENT_FRESH'} ${false} | ${'asdf'} | ${'ERROR'} | ${'CONTENT_STALE_ERROR'} `( @@ -84,6 +82,7 @@ describe('useReaderState', () => { activeSyncState: 'CHECKING', contentLoading: false, path: '', + buildLog: ['1', '2'], }; it('should return a copy of the state', () => { @@ -91,12 +90,14 @@ describe('useReaderState', () => { activeSyncState: 'CHECKING', contentLoading: false, path: '/', + buildLog: ['1', '2'], }); expect(oldState).toEqual({ activeSyncState: 'CHECKING', contentLoading: false, path: '', + buildLog: ['1', '2'], }); }); @@ -210,6 +211,33 @@ describe('useReaderState', () => { activeSyncState: 'BUILDING', }); }); + + it('should clear buildLog on "CHECKING"', () => { + expect( + reducer(oldState, { + type: 'sync', + state: 'CHECKING', + }), + ).toEqual({ + ...oldState, + activeSyncState: 'CHECKING', + buildLog: [], + }); + }); + }); + + describe('"buildLog" action', () => { + it('should work', () => { + expect( + reducer(oldState, { + type: 'buildLog', + log: 'Another Line', + }), + ).toEqual({ + ...oldState, + buildLog: ['1', '2', 'Another Line'], + }); + }); }); }); @@ -229,7 +257,10 @@ describe('useReaderState', () => { expect(result.current).toEqual({ state: 'CHECKING', content: undefined, - errorMessage: '', + contentErrorMessage: undefined, + syncErrorMessage: undefined, + buildLog: [], + contentReload: expect.any(Function), }); await waitForValueToChange(() => result.current.state); @@ -237,18 +268,24 @@ describe('useReaderState', () => { expect(result.current).toEqual({ state: 'CONTENT_FRESH', content: 'my content', - errorMessage: '', + contentErrorMessage: undefined, + syncErrorMessage: undefined, + buildLog: [], + contentReload: expect.any(Function), }); expect(techdocsStorageApi.getEntityDocs).toBeCalledWith( { kind: 'Component', namespace: 'default', name: 'backstage' }, '/example', ); - expect(techdocsStorageApi.syncEntityDocs).toBeCalledWith({ - kind: 'Component', - namespace: 'default', - name: 'backstage', - }); + expect(techdocsStorageApi.syncEntityDocs).toBeCalledWith( + { + kind: 'Component', + namespace: 'default', + name: 'backstage', + }, + expect.any(Function), + ); }); }); @@ -259,10 +296,14 @@ describe('useReaderState', () => { await new Promise(resolve => setTimeout(resolve, 500)); return 'my content'; }); - techdocsStorageApi.syncEntityDocs.mockImplementation(async () => { - await new Promise(resolve => setTimeout(resolve, 1100)); - return 'updated'; - }); + techdocsStorageApi.syncEntityDocs.mockImplementation( + async (_, logHandler) => { + logHandler?.call(this, 'Line 1'); + logHandler?.call(this, 'Line 2'); + await new Promise(resolve => setTimeout(resolve, 1100)); + return 'updated'; + }, + ); await act(async () => { const { result, waitForValueToChange } = await renderHook( @@ -273,7 +314,10 @@ describe('useReaderState', () => { expect(result.current).toEqual({ state: 'CHECKING', content: undefined, - errorMessage: '', + contentErrorMessage: undefined, + syncErrorMessage: undefined, + buildLog: [], + contentReload: expect.any(Function), }); await waitForValueToChange(() => result.current.state); @@ -281,7 +325,10 @@ describe('useReaderState', () => { expect(result.current).toEqual({ state: 'INITIAL_BUILD', content: undefined, - errorMessage: ' Load error: NotFoundError: Page Not Found', + contentErrorMessage: 'NotFoundError: Page Not Found', + syncErrorMessage: undefined, + buildLog: ['Line 1', 'Line 2'], + contentReload: expect.any(Function), }); await waitForValueToChange(() => result.current.state); @@ -289,7 +336,10 @@ describe('useReaderState', () => { expect(result.current).toEqual({ state: 'CHECKING', content: undefined, - errorMessage: '', + contentErrorMessage: undefined, + syncErrorMessage: undefined, + buildLog: [], + contentReload: expect.any(Function), }); await waitForValueToChange(() => result.current.state); @@ -297,7 +347,10 @@ describe('useReaderState', () => { expect(result.current).toEqual({ state: 'CONTENT_FRESH', content: 'my content', - errorMessage: '', + contentErrorMessage: undefined, + syncErrorMessage: undefined, + buildLog: [], + contentReload: expect.any(Function), }); expect(techdocsStorageApi.getEntityDocs).toBeCalledTimes(2); @@ -306,20 +359,32 @@ describe('useReaderState', () => { '/example', ); expect(techdocsStorageApi.syncEntityDocs).toBeCalledTimes(1); - expect(techdocsStorageApi.syncEntityDocs).toBeCalledWith({ - kind: 'Component', - namespace: 'default', - name: 'backstage', - }); + expect(techdocsStorageApi.syncEntityDocs).toBeCalledWith( + { + kind: 'Component', + namespace: 'default', + name: 'backstage', + }, + expect.any(Function), + ); }); }); it('should handle stale content', async () => { - techdocsStorageApi.getEntityDocs.mockResolvedValue('my content'); - techdocsStorageApi.syncEntityDocs.mockImplementation(async () => { - await new Promise(resolve => setTimeout(resolve, 1100)); - return 'updated'; - }); + techdocsStorageApi.getEntityDocs + .mockResolvedValueOnce('my content') + .mockImplementationOnce(async () => { + await new Promise(resolve => setTimeout(resolve, 1100)); + return 'my new content'; + }); + techdocsStorageApi.syncEntityDocs.mockImplementation( + async (_, logHandler) => { + logHandler?.call(this, 'Line 1'); + logHandler?.call(this, 'Line 2'); + await new Promise(resolve => setTimeout(resolve, 1100)); + return 'updated'; + }, + ); await act(async () => { const { result, waitForValueToChange } = await renderHook( @@ -330,7 +395,10 @@ describe('useReaderState', () => { expect(result.current).toEqual({ state: 'CHECKING', content: undefined, - errorMessage: '', + contentErrorMessage: undefined, + syncErrorMessage: undefined, + buildLog: [], + contentReload: expect.any(Function), }); // the content is returned but the sync is in progress @@ -338,7 +406,10 @@ describe('useReaderState', () => { expect(result.current).toEqual({ state: 'CONTENT_FRESH', content: 'my content', - errorMessage: '', + contentErrorMessage: undefined, + syncErrorMessage: undefined, + buildLog: ['Line 1', 'Line 2'], + contentReload: expect.any(Function), }); // the sync takes longer than 1 seconds so the refreshing state starts @@ -346,62 +417,61 @@ describe('useReaderState', () => { expect(result.current).toEqual({ state: 'CONTENT_STALE_REFRESHING', content: 'my content', - errorMessage: '', + contentErrorMessage: undefined, + syncErrorMessage: undefined, + buildLog: ['Line 1', 'Line 2'], + contentReload: expect.any(Function), }); - // the content is up-to-date + // the content is updated but not yet displayed await waitForValueToChange(() => result.current.state); expect(result.current).toEqual({ state: 'CONTENT_STALE_READY', content: 'my content', - errorMessage: '', + contentErrorMessage: undefined, + syncErrorMessage: undefined, + buildLog: ['Line 1', 'Line 2'], + contentReload: expect.any(Function), }); - expect(techdocsStorageApi.getEntityDocs).toBeCalledWith( - { kind: 'Component', namespace: 'default', name: 'backstage' }, - '/example', - ); - expect(techdocsStorageApi.syncEntityDocs).toBeCalledWith({ - kind: 'Component', - namespace: 'default', - name: 'backstage', - }); - }); - }); - - it('should handle timed-out refresh', async () => { - techdocsStorageApi.getEntityDocs.mockResolvedValue('my content'); - techdocsStorageApi.syncEntityDocs.mockResolvedValue('timeout'); - - await act(async () => { - const { result, waitForValueToChange } = await renderHook( - () => useReaderState('Component', 'default', 'backstage', '/example'), - { wrapper: Wrapper }, - ); + // reload the content + result.current.contentReload(); + // the new content refresh is triggered + await waitForValueToChange(() => result.current.state); expect(result.current).toEqual({ state: 'CHECKING', content: undefined, - errorMessage: '', + contentErrorMessage: undefined, + syncErrorMessage: undefined, + buildLog: [], + contentReload: expect.any(Function), }); - // the content is returned but the sync is in progress + // the new content is loaded await waitForValueToChange(() => result.current.state); expect(result.current).toEqual({ - state: 'CONTENT_STALE_TIMEOUT', - content: 'my content', - errorMessage: '', + state: 'CONTENT_FRESH', + content: 'my new content', + contentErrorMessage: undefined, + syncErrorMessage: undefined, + buildLog: [], + contentReload: expect.any(Function), }); + expect(techdocsStorageApi.getEntityDocs).toBeCalledTimes(2); expect(techdocsStorageApi.getEntityDocs).toBeCalledWith( { kind: 'Component', namespace: 'default', name: 'backstage' }, '/example', ); - expect(techdocsStorageApi.syncEntityDocs).toBeCalledWith({ - kind: 'Component', - namespace: 'default', - name: 'backstage', - }); + expect(techdocsStorageApi.syncEntityDocs).toBeCalledWith( + { + kind: 'Component', + namespace: 'default', + name: 'backstage', + }, + expect.any(Function), + ); }); }); @@ -420,7 +490,10 @@ describe('useReaderState', () => { expect(result.current).toEqual({ state: 'CHECKING', content: undefined, - errorMessage: '', + contentErrorMessage: undefined, + syncErrorMessage: undefined, + buildLog: [], + contentReload: expect.any(Function), }); // the content loading threw an error @@ -428,18 +501,24 @@ describe('useReaderState', () => { expect(result.current).toEqual({ state: 'CONTENT_NOT_FOUND', content: undefined, - errorMessage: ' Load error: NotFoundError: Some error description', + contentErrorMessage: 'NotFoundError: Some error description', + syncErrorMessage: undefined, + buildLog: [], + contentReload: expect.any(Function), }); expect(techdocsStorageApi.getEntityDocs).toBeCalledWith( { kind: 'Component', namespace: 'default', name: 'backstage' }, '/example', ); - expect(techdocsStorageApi.syncEntityDocs).toBeCalledWith({ - kind: 'Component', - namespace: 'default', - name: 'backstage', - }); + expect(techdocsStorageApi.syncEntityDocs).toBeCalledWith( + { + kind: 'Component', + namespace: 'default', + name: 'backstage', + }, + expect.any(Function), + ); }); }); }); diff --git a/plugins/techdocs/src/reader/components/useReaderState.ts b/plugins/techdocs/src/reader/components/useReaderState.ts index ac5145ee49..178c0746bf 100644 --- a/plugins/techdocs/src/reader/components/useReaderState.ts +++ b/plugins/techdocs/src/reader/components/useReaderState.ts @@ -14,10 +14,10 @@ * limitations under the License. */ +import { useApi } from '@backstage/core-plugin-api'; import { useEffect, useMemo, useReducer, useRef } from 'react'; import { useAsync, useAsyncRetry } from 'react-use'; import { techdocsStorageApiRef } from '../../api'; -import { useApi } from '@backstage/core-plugin-api'; /** * A state representation that is used to configure the UI of @@ -35,9 +35,6 @@ type ContentStateTypes = /** There is content, but after a reload, the content will be different */ | 'CONTENT_STALE_READY' - /** There is content, the backend tried to update it, but it took too long */ - | 'CONTENT_STALE_TIMEOUT' - /** There is content, the backend tried to update it, but failed */ | 'CONTENT_STALE_ERROR' @@ -93,11 +90,6 @@ export function calculateDisplayState({ return 'CONTENT_STALE_READY'; } - // the build timed out, but the content is still stale - if (activeSyncState === 'BUILD_TIMED_OUT') { - return 'CONTENT_STALE_TIMEOUT'; - } - // the build failed, but the content is still stale if (activeSyncState === 'ERROR') { return 'CONTENT_STALE_ERROR'; @@ -127,9 +119,6 @@ type SyncStates = */ | 'BUILD_READY_RELOAD' - /** Building the documentation timed out */ - | 'BUILD_TIMED_OUT' - /** No need for a sync. The content was already up-to-date. */ | 'UP_TO_DATE' @@ -148,7 +137,8 @@ type ReducerActions = contentLoading?: true; contentError?: Error; } - | { type: 'navigate'; path: string }; + | { type: 'navigate'; path: string } + | { type: 'buildLog'; log: string }; type ReducerState = { /** @@ -172,6 +162,11 @@ type ReducerState = { contentError?: Error; syncError?: Error; + + /** + * A list of log messages that were emitted by the build process. + */ + buildLog: string[]; }; export function reducer( @@ -182,6 +177,11 @@ export function reducer( switch (action.type) { case 'sync': + // reset the build log when a new check starts + if (action.state === 'CHECKING') { + newState.buildLog = []; + } + newState.activeSyncState = action.state; newState.syncError = action.syncError; break; @@ -196,6 +196,10 @@ export function reducer( newState.path = action.path; break; + case 'buildLog': + newState.buildLog = newState.buildLog.concat(action.log); + break; + default: throw new Error(); } @@ -206,6 +210,7 @@ export function reducer( ['content', 'navigate'].includes(action.type) ) { newState.activeSyncState = 'UP_TO_DATE'; + newState.buildLog = []; } return newState; @@ -216,11 +221,19 @@ export function useReaderState( namespace: string, name: string, path: string, -): { state: ContentStateTypes; content?: string; errorMessage?: string } { +): { + state: ContentStateTypes; + contentReload: () => void; + content?: string; + contentErrorMessage?: string; + syncErrorMessage?: string; + buildLog: string[]; +} { const [state, dispatch] = useReducer(reducer, { activeSyncState: 'CHECKING', path, contentLoading: true, + buildLog: [], }); const techdocsStorageApi = useApi(techdocsStorageApiRef); @@ -268,24 +281,38 @@ export function useReaderState( }, 1000); try { - const result = await techdocsStorageApi.syncEntityDocs({ - kind, - namespace, - name, - }); + const result = await techdocsStorageApi.syncEntityDocs( + { + kind, + namespace, + name, + }, + log => { + dispatch({ type: 'buildLog', log }); + }, + ); - if (result === 'updated') { - // if there was no content prior to building, retry the loading - if (!contentRef.current.content) { - contentRef.current.reload(); - dispatch({ type: 'sync', state: 'BUILD_READY_RELOAD' }); - } else { - dispatch({ type: 'sync', state: 'BUILD_READY' }); - } - } else if (result === 'cached') { - dispatch({ type: 'sync', state: 'UP_TO_DATE' }); - } else { - dispatch({ type: 'sync', state: 'BUILD_TIMED_OUT' }); + switch (result) { + case 'updated': + // if there was no content prior to building, retry the loading + if (!contentRef.current.content) { + contentRef.current.reload(); + dispatch({ type: 'sync', state: 'BUILD_READY_RELOAD' }); + } else { + dispatch({ type: 'sync', state: 'BUILD_READY' }); + } + break; + case 'cached': + dispatch({ type: 'sync', state: 'UP_TO_DATE' }); + break; + + default: + dispatch({ + type: 'sync', + state: 'ERROR', + syncError: new Error('Unexpected return state'), + }); + break; } } catch (e) { dispatch({ type: 'sync', state: 'ERROR', syncError: e }); @@ -305,19 +332,12 @@ export function useReaderState( [state.activeSyncState, state.content, state.contentLoading], ); - const errorMessage = useMemo(() => { - let errMessage = ''; - if (state.contentError) { - errMessage += ` Load error: ${state.contentError}`; - } - if (state.syncError) errMessage += ` Build error: ${state.syncError}`; - - return errMessage; - }, [state.syncError, state.contentError]); - return { state: displayState, + contentReload, content: state.content, - errorMessage, + contentErrorMessage: state.contentError?.toString(), + syncErrorMessage: state.syncError?.toString(), + buildLog: state.buildLog, }; } diff --git a/yarn.lock b/yarn.lock index 9f263eae28..76facd5001 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5625,6 +5625,11 @@ resolved "https://registry.npmjs.org/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f" integrity sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw== +"@types/eventsource@^1.1.5": + version "1.1.5" + resolved "https://registry.npmjs.org/@types/eventsource/-/eventsource-1.1.5.tgz#408e9b45efb176c8bea672ab58c81e7ab00d24bc" + integrity sha512-BA9q9uC2PAMkUS7DunHTxWZZaVpeNzDG8lkBxcKwzKJClfDQ4Z59/Csx7HSH/SIqFN2JWh0tAKAM6k/wRR0OZg== + "@types/express-serve-static-core@*", "@types/express-serve-static-core@^4.17.18", "@types/express-serve-static-core@^4.17.21", "@types/express-serve-static-core@^4.17.5": version "4.17.21" resolved "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.21.tgz#a427278e106bca77b83ad85221eae709a3414d42" @@ -12309,6 +12314,13 @@ eventsource@^1.0.5, eventsource@^1.0.7: dependencies: original "^1.0.0" +eventsource@^1.1.0: + version "1.1.0" + resolved "https://registry.npmjs.org/eventsource/-/eventsource-1.1.0.tgz#00e8ca7c92109e94b0ddf32dac677d841028cfaf" + integrity sha512-VSJjT5oCNrFvCS6igjzPAt5hBzQ2qPBFIbJ03zLI9SE0mxwZpMw6BfJrbFHm1a141AavMEB8JHmBhWAd66PfCg== + dependencies: + original "^1.0.0" + evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz#7fcbdb198dc71959432efe13842684e0525acb02"