From f097c32b59116e0399a46b9470e20e849afd9000 Mon Sep 17 00:00:00 2001 From: Sebastian Qvarfordt Date: Fri, 18 Sep 2020 14:02:35 +0200 Subject: [PATCH] TechDocs: Use a flag to determine if we should use a local mkdocs or techdocs-container (#2503) * Use a flag to determine wether to use techdocs-container or local install of mkdocs * Updated techdocs generator to look at app-config string instead of argument to decide how to run the generator * Removed console log... * Reverted scaffolder file that was accidentally committed. * Fixed lint issues * Added config to create-app template --- app-config.yaml | 2 + docs/features/techdocs/getting-started.md | 24 ++++++ packages/backend/src/plugins/techdocs.ts | 2 +- .../templates/default-app/app-config.yaml.hbs | 2 + .../packages/backend/src/plugins/techdocs.ts | 2 +- .../src/service/standaloneServer.ts | 5 +- .../stages/generate/generators.test.ts | 6 +- .../src/techdocs/stages/generate/techdocs.ts | 84 +++++++++++++------ 8 files changed, 97 insertions(+), 30 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index e018eef87e..5ca76ce789 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -35,6 +35,8 @@ organization: techdocs: storageUrl: http://localhost:7000/techdocs/static/docs requestUrl: http://localhost:7000/techdocs/docs + generators: + techdocs: 'docker' sentry: organization: spotify diff --git a/docs/features/techdocs/getting-started.md b/docs/features/techdocs/getting-started.md index d8c836c970..b9d215fd8a 100644 --- a/docs/features/techdocs/getting-started.md +++ b/docs/features/techdocs/getting-started.md @@ -71,6 +71,30 @@ building and publishing of your documentation, you want to change the `requestUrl` to point to your storage. In this case `storageUrl` is not required. +### Disable Docker in Docker situation (Optional) + +The TechDocs backend plugin runs a docker container with mkdocs to generate the +frontend of the docs from source files (Markdown). If you are deploying +Backstage using Docker, this will mean that your Backstage Docker container will +try to run another Docker container for TechDocs backend. + +To avoid this problem, we have a configuration available. You can set a value in +your `app-config.yaml` that tells the techdocs generator if it should run the +`local` mkdocs or run it from `docker`. This defaults to running as `docker` if +no config is provided. + +```yaml +techdocs: + generators: + techdocs: local +``` + +Setting `generators.techdocs` to `local` means you will have to make sure your +environment is compatible with techdocs. You will have to install the +`mkdocs-techdocs-container` and 'mkdocs' package from pip, as well as graphviz +and plantuml from your package manager. This has only been tested with python +3.7 and python 3.8. + ## Run Backstage locally Change folder to `/packages/backend` and run the diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index 7364cc4d83..58dca83b43 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -31,7 +31,7 @@ export default async function createPlugin({ config, }: PluginEnvironment) { const generators = new Generators(); - const techdocsGenerator = new TechdocsGenerator(logger); + const techdocsGenerator = new TechdocsGenerator(logger, config); generators.register('techdocs', techdocsGenerator); const preparers = new Preparers(); diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index ecac0613d9..d8d5adae37 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -46,6 +46,8 @@ proxy: techdocs: storageUrl: http://localhost:7000/techdocs/static/docs requestUrl: http://localhost:7000/techdocs/docs + generators: + techdocs: 'docker' lighthouse: baseUrl: http://localhost:3003 diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts index dfd9eb5e33..9c7de3512b 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts @@ -15,7 +15,7 @@ export default async function createPlugin({ config, }: PluginEnvironment) { const generators = new Generators(); - const techdocsGenerator = new TechdocsGenerator(logger); + const techdocsGenerator = new TechdocsGenerator(logger, config); generators.register('techdocs', techdocsGenerator); diff --git a/plugins/techdocs-backend/src/service/standaloneServer.ts b/plugins/techdocs-backend/src/service/standaloneServer.ts index 93233af4c0..7a39742b80 100644 --- a/plugins/techdocs-backend/src/service/standaloneServer.ts +++ b/plugins/techdocs-backend/src/service/standaloneServer.ts @@ -38,6 +38,7 @@ export async function startStandaloneServer( options: ServerOptions, ): Promise { const logger = options.logger.child({ service: 'techdocs-backend' }); + const config = ConfigReader.fromConfigs([]); logger.debug('Creating application...'); const preparers = new Preparers(); @@ -45,7 +46,7 @@ export async function startStandaloneServer( preparers.register('dir', directoryPreparer); const generators = new Generators(); - const techdocsGenerator = new TechdocsGenerator(logger); + const techdocsGenerator = new TechdocsGenerator(logger, config); generators.register('techdocs', techdocsGenerator); const publisher = new LocalPublish(logger); @@ -59,7 +60,7 @@ export async function startStandaloneServer( logger, publisher, dockerClient, - config: ConfigReader.fromConfigs([]), + config, }); const service = createServiceBuilder(module) .enableCors({ origin: 'http://localhost:3000' }) diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/generators.test.ts b/plugins/techdocs-backend/src/techdocs/stages/generate/generators.test.ts index b339aef9f8..a9303c3794 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/generate/generators.test.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/generate/generators.test.ts @@ -16,6 +16,7 @@ import { Generators, TechdocsGenerator } from './'; import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; const logger = getVoidLogger(); @@ -38,7 +39,10 @@ describe('generators', () => { it('should return correct registered generator', async () => { const generators = new Generators(); - const techdocs = new TechdocsGenerator(logger); + const techdocs = new TechdocsGenerator( + logger, + ConfigReader.fromConfigs([]), + ); generators.register('techdocs', techdocs); diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts b/plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts index cb89b42e38..ac73963f9a 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts @@ -18,6 +18,8 @@ import fs from 'fs-extra'; import path from 'path'; import os from 'os'; import { Logger } from 'winston'; +import { PassThrough } from 'stream'; +import { Config } from '@backstage/config'; import { GeneratorBase, @@ -26,18 +28,39 @@ import { } from './types'; import { runDockerContainer, runCommand } from './helpers'; -const commandExists = require('command-exists-promise'); +type TechdocsGeneratorOptions = { + // This option enables users to configure if they want to use TechDocs container + // or generate without the container. + // This is used to avoid running into Docker in Docker environment. + runGeneratorIn: string; +}; + +const createStream = (): [string[], PassThrough] => { + const log = [] as Array; + + const stream = new PassThrough(); + stream.on('data', chunk => { + const textValue = chunk.toString().trim(); + if (textValue?.length > 1) log.push(textValue); + }); + + return [log, stream]; +}; export class TechdocsGenerator implements GeneratorBase { private readonly logger: Logger; + private readonly options: TechdocsGeneratorOptions; - constructor(logger: Logger) { + constructor(logger: Logger, config: Config) { this.logger = logger; + this.options = { + runGeneratorIn: + config.getOptionalString('techdocs.generators.techdocs') ?? 'docker', + }; } public async run({ directory, - logStream, dockerClient, }: GeneratorRunOptions): Promise { const tmpdirPath = os.tmpdir(); @@ -46,35 +69,46 @@ export class TechdocsGenerator implements GeneratorBase { const resultDir = fs.mkdtempSync( path.join(tmpdirResolvedPath, 'techdocs-tmp-'), ); + const [log, logStream] = createStream(); try { - const mkdocsInstalled = await commandExists('mkdocs'); - if (mkdocsInstalled) { - await runCommand({ - command: 'mkdocs', - args: ['build', '-d', resultDir, '-v'], - options: { - cwd: directory, - }, - logStream, - }); - } else { - await runDockerContainer({ - imageName: 'spotify/techdocs', - args: ['build', '-d', '/result'], - logStream, - docsDir: directory, - resultDir, - dockerClient, - }); - this.logger.info( - `[TechDocs]: Successfully generated docs from ${directory} into ${resultDir}`, - ); + switch (this.options.runGeneratorIn) { + case 'local': + await runCommand({ + command: 'mkdocs', + args: ['build', '-d', resultDir, '-v'], + options: { + cwd: directory, + }, + logStream, + }); + this.logger.info( + `[TechDocs]: Successfully generated docs from ${directory} into ${resultDir} using local mkdocs`, + ); + break; + case 'docker': + await runDockerContainer({ + imageName: 'spotify/techdocs', + args: ['build', '-d', '/result'], + logStream, + docsDir: directory, + resultDir, + dockerClient, + }); + this.logger.info( + `[TechDocs]: Successfully generated docs from ${directory} into ${resultDir} using techdocs-container`, + ); + break; + default: + throw new Error( + `Invalid config value "${this.options.runGeneratorIn}" provided in 'techdocs.generators.techdocs'.`, + ); } } catch (error) { this.logger.debug( `[TechDocs]: Failed to generate docs from ${directory} into ${resultDir}`, ); + this.logger.debug(`[TechDocs]: Build failed with error: ${log}`); throw new Error( `Failed to generate docs from ${directory} into ${resultDir} with error ${error.message}`, );