From e09cff1682c481f9378ba144ff88fd537671fc45 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 2 Dec 2020 11:45:19 +0100 Subject: [PATCH] Abstract Publisher APIs from the router 1. Move fetchMetadata and serve static files to local publisher 2. Create factory class to create publishers based on app cofig --- packages/backend/src/plugins/techdocs.ts | 2 +- .../packages/backend/src/plugins/techdocs.ts | 2 +- packages/techdocs-common/package.json | 4 +- .../src/stages/publish/index.ts | 3 +- .../src/stages/publish/local.test.ts | 14 ++- .../src/stages/publish/local.ts | 83 +++++++++++----- .../src/stages/publish/publish.ts | 98 ++++--------------- .../test-component-folder-AP2Wti/mock-file | 0 .../src/stages/publish/types.ts | 42 +++++++- .../techdocs-backend/src/service/helpers.ts | 17 +++- .../techdocs-backend/src/service/router.ts | 42 +++----- .../src/service/standaloneServer.ts | 2 +- yarn.lock | 9 ++ 13 files changed, 173 insertions(+), 145 deletions(-) delete mode 100644 packages/techdocs-common/src/stages/publish/test-component-folder-AP2Wti/mock-file diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index 346ea37bac..4720b2568e 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -49,7 +49,7 @@ export default async function createPlugin({ const urlPreparer = new UrlPreparer(reader, logger); preparers.register('url', urlPreparer); - const publisher = new Publisher(logger, config, discovery); + const publisher = Publisher.fromConfig(config, logger, discovery); const dockerClient = new Docker(); diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts index 61af70d981..1bbb5ff24b 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts @@ -28,7 +28,7 @@ export default async function createPlugin({ preparers.register('github', commonGitPreparer); preparers.register('gitlab', commonGitPreparer); - const publisher = new Publisher(logger, config, discovery); + const publisher = Publisher.fromConfig(config, logger, discovery); const dockerClient = new Docker(); diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 6edb1fc589..cd5e593ad1 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -39,14 +39,16 @@ "@backstage/backend-common": "^0.3.2", "@backstage/catalog-model": "^0.3.1", "@backstage/config": "^0.1.1", + "@types/dockerode": "^2.5.34", + "@types/express-serve-static-core": "^4.17.14", "cross-fetch": "^3.0.6", "dockerode": "^3.2.1", + "express": "^4.17.1", "fs-extra": "^9.0.1", "git-url-parse": "^11.4.0", "js-yaml": "^3.14.0", "mock-fs": "^4.13.0", "nodegit": "^0.27.0", - "@types/dockerode": "^2.5.34", "winston": "^3.2.1" }, "devDependencies": { diff --git a/packages/techdocs-common/src/stages/publish/index.ts b/packages/techdocs-common/src/stages/publish/index.ts index 9a21217083..494efe5ac1 100644 --- a/packages/techdocs-common/src/stages/publish/index.ts +++ b/packages/techdocs-common/src/stages/publish/index.ts @@ -14,5 +14,4 @@ * limitations under the License. */ export { Publisher } from './publish'; -export type { PublisherBase } from './publish'; -export type { PublisherType } from './types'; +export type { PublisherBase, PublisherType } from './types'; diff --git a/packages/techdocs-common/src/stages/publish/local.test.ts b/packages/techdocs-common/src/stages/publish/local.test.ts index 793c2374d9..a790284c3f 100644 --- a/packages/techdocs-common/src/stages/publish/local.test.ts +++ b/packages/techdocs-common/src/stages/publish/local.test.ts @@ -21,6 +21,7 @@ import { getVoidLogger, PluginEndpointDiscovery, } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; import { LocalPublish } from './local'; const createMockEntity = (annotations = {}) => { @@ -45,7 +46,18 @@ describe('local publisher', () => { getExternalBaseUrl: jest.fn(), }; - const publisher = new LocalPublish(logger, testDiscovery); + const mockConfig = ConfigReader.fromConfigs([ + { + context: '', + data: { + techdocs: { + requestUrl: 'http://localhost:7000', + }, + }, + }, + ]); + + const publisher = new LocalPublish(mockConfig, logger, testDiscovery); const mockEntity = createMockEntity(); diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index 712bf8266d..c1be8641ef 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -13,50 +13,47 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import express from 'express'; +import * as expressServeStaticCore from 'express-serve-static-core'; // Type from express library +import fetch from 'cross-fetch'; import fs from 'fs-extra'; import { Logger } from 'winston'; -import { Entity } from '@backstage/catalog-model'; +import { EntityName } from '@backstage/catalog-model'; import { resolvePackagePath, PluginEndpointDiscovery, } from '@backstage/backend-common'; +import { Config } from '@backstage/config'; +import { PublisherBase, PublisherBaseParams } from './types'; -export type LocalPublishParams = { - entity: Entity; - directory: string; -}; +/* `remoteUrl` is the URL which serves files from the local publisher's static directory. */ +export type LocalPublishReturn = Promise<{ remoteUrl: string }>; -export type LocalPublishReturn = - | Promise<{ remoteUrl: string }> - | { remoteUrl: string }; +const staticDocsDir = resolvePackagePath( + '@backstage/plugin-techdocs-backend', + 'static/docs', +); /** - * Type for the local publisher which uses local filesystem to store the generated static files. - * - * It uses a directory called "static" at the root of techdocs-backend plugin. + * Local publisher which uses the local filesystem to store the generated static files. It uses a directory + * called "static" at the root of techdocs-backend plugin. */ -export interface LocalPublisher { - /** - * Store the generated files inside a static folder in local filesystem. - * - * @param {LocalPublishParams} opts Object containing the entity from the service - * catalog, and the directory that contains the generated static files from TechDocs. - * @returns {LocalPublishReturn} Either a promise or an object with `remoteUrl` which is the URL - * which serves files from the local publisher's static directory. - */ - publish(opts: LocalPublishParams): LocalPublishReturn; -} - -export class LocalPublish { +export class LocalPublish implements PublisherBase { + private readonly config: Config; private readonly logger: Logger; private readonly discovery: PluginEndpointDiscovery; - constructor(logger: Logger, discovery: PluginEndpointDiscovery) { + constructor( + config: Config, + logger: Logger, + discovery: PluginEndpointDiscovery, + ) { + this.config = config; this.logger = logger; this.discovery = discovery; } - publish({ entity, directory }: LocalPublishParams): LocalPublishReturn { + publish({ entity, directory }: PublisherBaseParams): LocalPublishReturn { const entityNamespace = entity.metadata.namespace ?? 'default'; const publishDir = resolvePackagePath( @@ -94,4 +91,38 @@ export class LocalPublish { }); }); } + + fetchTechDocsMetadata( + entityName: EntityName, + ): Promise<{ techdocsMetadataJson: string }> { + return new Promise((resolve, reject) => { + this.discovery.getBaseUrl('techdocs').then(techdocsApiUrl => { + const storageUrl = new URL( + new URL(this.config.getString('techdocs.storageUrl')).pathname, + techdocsApiUrl, + ).toString(); + + const path = `${entityName.kind}/${entityName.namespace}/${entityName.name}`; + const metadataURL = `${storageUrl}/${path}/techdocs_metadata.json`; + fetch(metadataURL) + .then(response => + response + .json() + .then(techdocsMetadataJson => resolve(techdocsMetadataJson)) + .catch(err => { + reject( + `Unable to parse metadata JSON for ${path}. Error: ${err}`, + ); + }), + ) + .catch(err => { + reject(`Unable to fetch metadata for ${path}. Error ${err}`); + }); + }); + }); + } + + docsRouter(): expressServeStaticCore.Handler { + return express.static(staticDocsDir); + } } diff --git a/packages/techdocs-common/src/stages/publish/publish.ts b/packages/techdocs-common/src/stages/publish/publish.ts index bf01812fa2..ca3cb99498 100644 --- a/packages/techdocs-common/src/stages/publish/publish.ts +++ b/packages/techdocs-common/src/stages/publish/publish.ts @@ -16,96 +16,34 @@ import { Logger } from 'winston'; import { Config } from '@backstage/config'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; -import { Entity } from '@backstage/catalog-model'; -import { PublisherType } from './types'; + +import { PublisherType, PublisherBase } from './types'; import { LocalPublish } from './local'; -export type PublisherBaseParams = { - entity: Entity; - directory: string; -}; - -export type PublisherBaseReturn = - | Promise<{ remoteUrl: string }> - | { remoteUrl: string }; - /** - * Type for the publisher instance registered with backend which manages publishing of the - * generated static files after the prepare and generate steps of TechDocs. - * - * Depending upon the value of techdocs.publisher.type in app config, this instance creates - * and uses a publisher of the particular type i.e. local, google_gcs, aws_s3, etc. It defaults - * to use the local publisher. + * Factory class to create a TechDocs publisher based on defined publisher type in app config. + * Uses `techdocs.publisher.type`. */ -export interface PublisherBase { - /** - * Invoke the app's configured publisher's publish method. - * - * @param {PublisherBaseParams} opts Object containing the entity from the service - * catalog, and the directory that contains the generated static files from TechDocs. - */ - publish(opts: PublisherBaseParams): PublisherBaseReturn; - - /** - * Return true if local filesystem is being used to store generated files for TechDocs. - */ - isLocalPublisher(): boolean; - - /** - * Return true if an external cloud storage (GCS, S3, SFTP server, etc.) is being used to - * store generated files for TechDocs. - */ - isExternalPublisher(): boolean; -} - -export class Publisher implements PublisherBase { - private readonly logger: Logger; - private readonly config: Config; - private readonly discovery: PluginEndpointDiscovery; - private readonly publisherType: PublisherType; - private readonly publisher: any; - - constructor( - logger: Logger, +export class Publisher { + static fromConfig( config: Config, + logger: Logger, discovery: PluginEndpointDiscovery, - ) { - this.logger = logger; - this.config = config; - this.discovery = discovery; + ): PublisherBase { + const publisherType = (config.getOptionalString( + 'techdocs.publisher.type', + ) ?? 'local') as PublisherType; - this.publisherType = - (this.config.getOptionalString( - 'techdocs.publisher.type', - ) as PublisherType) ?? 'local'; - - switch (this.publisherType) { + switch (publisherType) { case 'google_gcs': - this.logger.info( - 'Creating Google Storage Bucket publisher for TechDocs', - ); - this.publisher = new LocalPublish(this.logger, this.discovery); - break; + logger.info('Creating Google Storage Bucket publisher for TechDocs'); + return new LocalPublish(config, logger, discovery); case 'local': - this.logger.info('Creating Local publisher for TechDocs'); - this.publisher = new LocalPublish(this.logger, this.discovery); - break; + logger.info('Creating Local publisher for TechDocs'); + return new LocalPublish(config, logger, discovery); default: - this.logger.info('Creating Local publisher for TechDocs'); - this.publisher = new LocalPublish(this.logger, this.discovery); - break; + logger.info('Creating Local publisher for TechDocs'); + return new LocalPublish(config, logger, discovery); } } - - publish({ entity, directory }: PublisherBaseParams): PublisherBaseReturn { - return this.publisher.publish({ entity, directory }); - } - - isLocalPublisher(): boolean { - return this.publisherType === 'local'; - } - - isExternalPublisher(): boolean { - return this.publisherType !== 'local'; - } } diff --git a/packages/techdocs-common/src/stages/publish/test-component-folder-AP2Wti/mock-file b/packages/techdocs-common/src/stages/publish/test-component-folder-AP2Wti/mock-file deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index 2c2348e37a..56f57c11c3 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -13,10 +13,48 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { Entity, EntityName } from '@backstage/catalog-model'; +// import serveStatic from 'serve-static'; +// import express from 'express'; +import * as express from 'express-serve-static-core'; /** - * Key for all the different types of TechDocs publishers available. + * Key for all the different types of TechDocs publishers that are supported. */ export type PublisherType = 'local' | 'google_gcs'; -export interface ExternalPublisher {} +export type PublisherBaseParams = { + entity: Entity; + /* The Path to the directory where the generated files are stored. */ + directory: string; +}; + +export type PublisherBaseReturn = Promise<{}>; + +/** + * Base class for a TechDocs publisher (e.g. Local, Google GCS Bucket, AWS S3, etc.) + * The publisher handles publishing of the generated static files after the prepare and generate steps of TechDocs. + * It also provides APIs to communicate with the storage service. + */ +export interface PublisherBase { + /** + * Store the generated static files onto a storage service (either local filesystem or external service). + * + * @param options Object containing the entity from the service + * catalog, and the directory that contains the generated static files from TechDocs. + */ + publish(options: PublisherBaseParams): PublisherBaseReturn; + + /** + * Retrieve TechDocs Metadata about a site e.g. name, contributors, last updated, etc. + * This API uses the techdocs_metadata.json file that co-exists along with the generated docs. + */ + fetchTechDocsMetadata( + entityName: EntityName, + ): Promise<{ techdocsMetadataJson: string }>; + + /** + * + */ + docsRouter(): express.Handler; +} diff --git a/plugins/techdocs-backend/src/service/helpers.ts b/plugins/techdocs-backend/src/service/helpers.ts index 876b80781e..51e7ed8ad2 100644 --- a/plugins/techdocs-backend/src/service/helpers.ts +++ b/plugins/techdocs-backend/src/service/helpers.ts @@ -15,7 +15,7 @@ */ import Docker from 'dockerode'; import { Logger } from 'winston'; -import { Entity } from '@backstage/catalog-model'; +import { Entity, EntityName } from '@backstage/catalog-model'; import { PreparerBuilder, PublisherBase, @@ -127,3 +127,18 @@ export class DocsBuilder { return false; } } + +/** + * Using the path of the TechDocs page URL, return a structured EntityName type object with namespace, + * kind and name of the Entity. + * @param {string} path Example: default/Component/documented-component + */ +export const getEntityNameFromUrlPath = (path: string): EntityName => { + const [kind, namespace, name] = path.split('/'); + + return { + kind, + namespace, + name, + }; +}; diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 0c5708e09f..dd084bde4b 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -26,12 +26,9 @@ import { PublisherBase, getLocationForEntity, } from '@backstage/techdocs-common'; -import { - PluginEndpointDiscovery, - resolvePackagePath, -} from '@backstage/backend-common'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; -import { DocsBuilder } from './helpers'; +import { DocsBuilder, getEntityNameFromUrlPath } from './helpers'; type RouterOptions = { preparers: PreparerBuilder; @@ -44,11 +41,6 @@ type RouterOptions = { dockerClient: Docker; }; -const staticDocsDir = resolvePackagePath( - '@backstage/plugin-techdocs-backend', - 'static/docs', -); - export async function createRouter({ preparers, generators, @@ -61,24 +53,18 @@ export async function createRouter({ const router = Router(); router.get('/metadata/techdocs/*', async (req, res) => { - let storageUrl = config.getString('techdocs.storageUrl'); - if (publisher.isLocalPublisher()) { - storageUrl = new URL( - new URL(storageUrl).pathname, - await discovery.getBaseUrl('techdocs'), - ).toString(); - } + // path is `:namespace/:kind:/:name` const { '0': path } = req.params; + const entityName = getEntityNameFromUrlPath(path); - const metadataURL = `${storageUrl}/${path}/techdocs_metadata.json`; - - try { - const techdocsMetadata = await (await fetch(metadataURL)).json(); - res.send(techdocsMetadata); - } catch (err) { - logger.info(`Unable to get metadata for ${path} with error ${err}`); - throw new Error(`Unable to get metadata for ${path} with error ${err}`); - } + publisher + .fetchTechDocsMetadata(entityName) + .then(techdocsMetadataJson => { + res.send(techdocsMetadataJson); + }) + .catch(reason => { + res.status(500).send(`Unable to get Metadata. Reason: ${reason}`); + }); }); router.get('/metadata/entity/:namespace/:kind/:name', async (req, res) => { @@ -139,9 +125,7 @@ export async function createRouter({ res.redirect(`${storageUrl}${req.path.replace('/docs', '')}`); }); - if (publisher.isLocalPublisher()) { - router.use('/static/docs', express.static(staticDocsDir)); - } + router.use('/static/docs', publisher.docsRouter()); return router; } diff --git a/plugins/techdocs-backend/src/service/standaloneServer.ts b/plugins/techdocs-backend/src/service/standaloneServer.ts index af3136dcda..c1c4dba8d1 100644 --- a/plugins/techdocs-backend/src/service/standaloneServer.ts +++ b/plugins/techdocs-backend/src/service/standaloneServer.ts @@ -64,7 +64,7 @@ export async function startStandaloneServer( const techdocsGenerator = new TechdocsGenerator(logger, config); generators.register('techdocs', techdocsGenerator); - const publisher = new Publisher(logger, config, discovery); + const publisher = Publisher.fromConfig(config, logger, discovery); const dockerClient = new Docker(); diff --git a/yarn.lock b/yarn.lock index 6f27d7028a..e031375fc0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5208,6 +5208,15 @@ "@types/qs" "*" "@types/range-parser" "*" +"@types/express-serve-static-core@^4.17.14": + version "4.17.14" + resolved "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.14.tgz#cabf91debeeb3cb04b798e2cff908864e89b6106" + integrity sha512-uFTLwu94TfUFMToXNgRZikwPuZdOtDgs3syBtAIr/OXorL1kJqUJT9qCLnRZ5KBOWfZQikQ2xKgR2tnDj1OgDA== + dependencies: + "@types/node" "*" + "@types/qs" "*" + "@types/range-parser" "*" + "@types/express-session@^1.17.2": version "1.17.2" resolved "https://registry.npmjs.org/@types/express-session/-/express-session-1.17.2.tgz#ed6a36dd9f267c7fef86004f653bfb9b5cea3c21"