From 9ebc89735160cce7b18c89d1a3bc69ceddf268d2 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 25 Nov 2020 01:56:55 +0100 Subject: [PATCH] TechDocs: Refactor Publisher to pave way for external publishers --- app-config.yaml | 5 + packages/backend/src/plugins/techdocs.ts | 4 +- .../packages/backend/src/plugins/techdocs.ts | 4 +- .../techdocs-backend/src/service/router.ts | 11 +- .../src/service/standaloneServer.ts | 17 ++- .../src/techdocs/stages/publish/index.ts | 5 +- .../src/techdocs/stages/publish/local.ts | 41 +++++-- .../src/techdocs/stages/publish/publish.ts | 111 ++++++++++++++++++ .../src/techdocs/stages/publish/types.ts | 18 +-- 9 files changed, 172 insertions(+), 44 deletions(-) create mode 100644 plugins/techdocs-backend/src/techdocs/stages/publish/publish.ts diff --git a/app-config.yaml b/app-config.yaml index 3b476981e4..f381f71ead 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -60,6 +60,11 @@ techdocs: requestUrl: http://localhost:7000/api/techdocs generators: techdocs: 'docker' + publisher: + # Ref: + type: 'local' # other options - google_gcs, aws_s3, azure_storage, etc. + # if type set to google_gcs + # google_token: 'abc' # with write access sentry: organization: my-company diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index de48280e64..3669911c0c 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -19,10 +19,10 @@ import { DirectoryPreparer, Preparers, Generators, - LocalPublish, TechdocsGenerator, CommonGitPreparer, UrlPreparer, + Publisher, } from '@backstage/plugin-techdocs-backend'; import { PluginEnvironment } from '../types'; import Docker from 'dockerode'; @@ -50,7 +50,7 @@ export default async function createPlugin({ const urlPreparer = new UrlPreparer(reader, logger); preparers.register('url', urlPreparer); - const publisher = new LocalPublish(logger, discovery); + const publisher = new Publisher(logger, config, discovery); const dockerClient = new Docker(); diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts index ac4d81a8e8..3b8df8ef21 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts @@ -4,7 +4,7 @@ import { CommonGitPreparer, Preparers, Generators, - LocalPublish, + Publisher, TechdocsGenerator, } from '@backstage/plugin-techdocs-backend'; import { PluginEnvironment } from '../types'; @@ -28,7 +28,7 @@ export default async function createPlugin({ preparers.register('github', commonGitPreparer); preparers.register('gitlab', commonGitPreparer); - const publisher = new LocalPublish(logger, discovery); + const publisher = new Publisher(logger, config, discovery); const dockerClient = new Docker(); diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 625252dd7e..61f0fa2e9f 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -20,12 +20,7 @@ import Knex from 'knex'; import fetch from 'cross-fetch'; import { Config } from '@backstage/config'; import Docker from 'dockerode'; -import { - GeneratorBuilder, - PreparerBuilder, - PublisherBase, - LocalPublish, -} from '../techdocs'; +import { GeneratorBuilder, PreparerBuilder, PublisherBase } from '../techdocs'; import { PluginEndpointDiscovery, resolvePackagePath, @@ -63,7 +58,7 @@ export async function createRouter({ router.get('/metadata/techdocs/*', async (req, res) => { let storageUrl = config.getString('techdocs.storageUrl'); - if (publisher instanceof LocalPublish) { + if (publisher.isLocalPublisher()) { storageUrl = new URL( new URL(storageUrl).pathname, await discovery.getBaseUrl('techdocs'), @@ -140,7 +135,7 @@ export async function createRouter({ res.redirect(`${storageUrl}${req.path.replace('/docs', '')}`); }); - if (publisher instanceof LocalPublish) { + if (publisher.isLocalPublisher()) { router.use('/static/docs', express.static(staticDocsDir)); } diff --git a/plugins/techdocs-backend/src/service/standaloneServer.ts b/plugins/techdocs-backend/src/service/standaloneServer.ts index 4082a36172..89b49467d4 100644 --- a/plugins/techdocs-backend/src/service/standaloneServer.ts +++ b/plugins/techdocs-backend/src/service/standaloneServer.ts @@ -27,7 +27,7 @@ import { DirectoryPreparer, Generators, TechdocsGenerator, - LocalPublish, + Publisher, } from '../techdocs'; import { ConfigReader } from '@backstage/config'; @@ -41,7 +41,18 @@ export async function startStandaloneServer( options: ServerOptions, ): Promise { const logger = options.logger.child({ service: 'techdocs-backend' }); - const config = ConfigReader.fromConfigs([]); + const config = ConfigReader.fromConfigs([ + { + context: '', + data: { + techdocs: { + publisher: { + type: 'local', + }, + }, + }, + }, + ]); const discovery = SingleHostDiscovery.fromConfig(config); logger.debug('Creating application...'); @@ -53,7 +64,7 @@ export async function startStandaloneServer( const techdocsGenerator = new TechdocsGenerator(logger, config); generators.register('techdocs', techdocsGenerator); - const publisher = new LocalPublish(logger, discovery); + const publisher = new Publisher(logger, config, discovery); const dockerClient = new Docker(); diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/index.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/index.ts index 0029ea5c4e..9a21217083 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/publish/index.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/index.ts @@ -13,5 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { LocalPublish } from './local'; -export type { PublisherBase } from './types'; +export { Publisher } from './publish'; +export type { PublisherBase } from './publish'; +export type { PublisherType } from './types'; diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts index 464c4b90a0..712bf8266d 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts @@ -16,13 +16,38 @@ import fs from 'fs-extra'; import { Logger } from 'winston'; import { Entity } from '@backstage/catalog-model'; -import { PublisherBase } from './types'; import { resolvePackagePath, PluginEndpointDiscovery, } from '@backstage/backend-common'; -export class LocalPublish implements PublisherBase { +export type LocalPublishParams = { + entity: Entity; + directory: string; +}; + +export type LocalPublishReturn = + | Promise<{ remoteUrl: string }> + | { remoteUrl: string }; + +/** + * Type for the local publisher which uses local filesystem to store the generated static files. + * + * It uses a directory called "static" at the root of techdocs-backend plugin. + */ +export interface LocalPublisher { + /** + * Store the generated files inside a static folder in local filesystem. + * + * @param {LocalPublishParams} opts Object containing the entity from the service + * catalog, and the directory that contains the generated static files from TechDocs. + * @returns {LocalPublishReturn} Either a promise or an object with `remoteUrl` which is the URL + * which serves files from the local publisher's static directory. + */ + publish(opts: LocalPublishParams): LocalPublishReturn; +} + +export class LocalPublish { private readonly logger: Logger; private readonly discovery: PluginEndpointDiscovery; @@ -31,17 +56,7 @@ export class LocalPublish implements PublisherBase { this.discovery = discovery; } - publish({ - entity, - directory, - }: { - entity: Entity; - directory: string; - }): - | Promise<{ - remoteUrl: string; - }> - | { remoteUrl: string } { + publish({ entity, directory }: LocalPublishParams): LocalPublishReturn { const entityNamespace = entity.metadata.namespace ?? 'default'; const publishDir = resolvePackagePath( diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/publish.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/publish.ts new file mode 100644 index 0000000000..bf01812fa2 --- /dev/null +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/publish.ts @@ -0,0 +1,111 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Logger } from 'winston'; +import { Config } from '@backstage/config'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; +import { PublisherType } from './types'; +import { LocalPublish } from './local'; + +export type PublisherBaseParams = { + entity: Entity; + directory: string; +}; + +export type PublisherBaseReturn = + | Promise<{ remoteUrl: string }> + | { remoteUrl: string }; + +/** + * Type for the publisher instance registered with backend which manages publishing of the + * generated static files after the prepare and generate steps of TechDocs. + * + * Depending upon the value of techdocs.publisher.type in app config, this instance creates + * and uses a publisher of the particular type i.e. local, google_gcs, aws_s3, etc. It defaults + * to use the local publisher. + */ +export interface PublisherBase { + /** + * Invoke the app's configured publisher's publish method. + * + * @param {PublisherBaseParams} opts Object containing the entity from the service + * catalog, and the directory that contains the generated static files from TechDocs. + */ + publish(opts: PublisherBaseParams): PublisherBaseReturn; + + /** + * Return true if local filesystem is being used to store generated files for TechDocs. + */ + isLocalPublisher(): boolean; + + /** + * Return true if an external cloud storage (GCS, S3, SFTP server, etc.) is being used to + * store generated files for TechDocs. + */ + isExternalPublisher(): boolean; +} + +export class Publisher implements PublisherBase { + private readonly logger: Logger; + private readonly config: Config; + private readonly discovery: PluginEndpointDiscovery; + private readonly publisherType: PublisherType; + private readonly publisher: any; + + constructor( + logger: Logger, + config: Config, + discovery: PluginEndpointDiscovery, + ) { + this.logger = logger; + this.config = config; + this.discovery = discovery; + + this.publisherType = + (this.config.getOptionalString( + 'techdocs.publisher.type', + ) as PublisherType) ?? 'local'; + + switch (this.publisherType) { + case 'google_gcs': + this.logger.info( + 'Creating Google Storage Bucket publisher for TechDocs', + ); + this.publisher = new LocalPublish(this.logger, this.discovery); + break; + case 'local': + this.logger.info('Creating Local publisher for TechDocs'); + this.publisher = new LocalPublish(this.logger, this.discovery); + break; + default: + this.logger.info('Creating Local publisher for TechDocs'); + this.publisher = new LocalPublish(this.logger, this.discovery); + break; + } + } + + publish({ entity, directory }: PublisherBaseParams): PublisherBaseReturn { + return this.publisher.publish({ entity, directory }); + } + + isLocalPublisher(): boolean { + return this.publisherType === 'local'; + } + + isExternalPublisher(): boolean { + return this.publisherType !== 'local'; + } +} diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/types.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/types.ts index ca85d9f56e..2c2348e37a 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/publish/types.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/types.ts @@ -13,20 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; /** - * Publisher is in charge of taking a folder created by - * the builder, and pushing it to storage + * Key for all the different types of TechDocs publishers available. */ -export type PublisherBase = { - /** - * - * @param opts object containing the entity from the service - * catalog, and the directory that has been generated - */ - publish(opts: { - entity: Entity; - directory: string; - }): Promise<{ remoteUrl: string }> | { remoteUrl: string }; -}; +export type PublisherType = 'local' | 'google_gcs'; + +export interface ExternalPublisher {}