TechDocs: Use a flag to determine if we should use a local mkdocs or techdocs-container (#2503)

* Use a flag to determine wether to use techdocs-container or local install of mkdocs

* Updated techdocs generator to look at app-config string instead of argument to decide how to run the generator

* Removed console log...

* Reverted scaffolder file that was accidentally committed.

* Fixed lint issues

* Added config to create-app template

app-config.yaml

@@ -35,6 +35,8 @@ organization:
 techdocs:
   storageUrl: http://localhost:7000/techdocs/static/docs
   requestUrl: http://localhost:7000/techdocs/docs
+  generators:
+    techdocs: 'docker'
 
 sentry:
   organization: spotify

docs/features/techdocs/getting-started.md

@@ -71,6 +71,30 @@ building and publishing of your documentation, you want to change the
 `requestUrl` to point to your storage. In this case `storageUrl` is not
 required.
 
+### Disable Docker in Docker situation (Optional)
+
+The TechDocs backend plugin runs a docker container with mkdocs to generate the
+frontend of the docs from source files (Markdown). If you are deploying
+Backstage using Docker, this will mean that your Backstage Docker container will
+try to run another Docker container for TechDocs backend.
+
+To avoid this problem, we have a configuration available. You can set a value in
+your `app-config.yaml` that tells the techdocs generator if it should run the
+`local` mkdocs or run it from `docker`. This defaults to running as `docker` if
+no config is provided.
+
+```yaml
+techdocs:
+  generators:
+    techdocs: local
+```
+
+Setting `generators.techdocs` to `local` means you will have to make sure your
+environment is compatible with techdocs. You will have to install the
+`mkdocs-techdocs-container` and 'mkdocs' package from pip, as well as graphviz
+and plantuml from your package manager. This has only been tested with python
+3.7 and python 3.8.
+
 ## Run Backstage locally
 
 Change folder to `<backstage-project-root>/packages/backend` and run the

packages/backend/src/plugins/techdocs.ts

@@ -31,7 +31,7 @@ export default async function createPlugin({
   config,
 }: PluginEnvironment) {
   const generators = new Generators();
-  const techdocsGenerator = new TechdocsGenerator(logger);
+  const techdocsGenerator = new TechdocsGenerator(logger, config);
   generators.register('techdocs', techdocsGenerator);
 
   const preparers = new Preparers();

packages/create-app/templates/default-app/app-config.yaml.hbs

@@ -46,6 +46,8 @@ proxy:
 techdocs:
   storageUrl: http://localhost:7000/techdocs/static/docs
   requestUrl: http://localhost:7000/techdocs/docs
+  generators:
+    techdocs: 'docker'
 
 lighthouse:
   baseUrl: http://localhost:3003

packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts

@@ -15,7 +15,7 @@ export default async function createPlugin({
   config,
 }: PluginEnvironment) {
   const generators = new Generators();
-  const techdocsGenerator = new TechdocsGenerator(logger);
+  const techdocsGenerator = new TechdocsGenerator(logger, config);
 
   generators.register('techdocs', techdocsGenerator);
 

plugins/techdocs-backend/src/service/standaloneServer.ts

@@ -38,6 +38,7 @@ export async function startStandaloneServer(
   options: ServerOptions,
 ): Promise<Server> {
   const logger = options.logger.child({ service: 'techdocs-backend' });
+  const config = ConfigReader.fromConfigs([]);
 
   logger.debug('Creating application...');
   const preparers = new Preparers();
@@ -45,7 +46,7 @@ export async function startStandaloneServer(
   preparers.register('dir', directoryPreparer);
 
   const generators = new Generators();
-  const techdocsGenerator = new TechdocsGenerator(logger);
+  const techdocsGenerator = new TechdocsGenerator(logger, config);
   generators.register('techdocs', techdocsGenerator);
 
   const publisher = new LocalPublish(logger);
@@ -59,7 +60,7 @@ export async function startStandaloneServer(
     logger,
     publisher,
     dockerClient,
-    config: ConfigReader.fromConfigs([]),
+    config,
   });
   const service = createServiceBuilder(module)
     .enableCors({ origin: 'http://localhost:3000' })

plugins/techdocs-backend/src/techdocs/stages/generate/generators.test.ts

@@ -16,6 +16,7 @@
 
 import { Generators, TechdocsGenerator } from './';
 import { getVoidLogger } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
 
 const logger = getVoidLogger();
 
@@ -38,7 +39,10 @@ describe('generators', () => {
 
   it('should return correct registered generator', async () => {
     const generators = new Generators();
-    const techdocs = new TechdocsGenerator(logger);
+    const techdocs = new TechdocsGenerator(
+      logger,
+      ConfigReader.fromConfigs([]),
+    );
 
     generators.register('techdocs', techdocs);
 

plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts

@@ -18,6 +18,8 @@ import fs from 'fs-extra';
 import path from 'path';
 import os from 'os';
 import { Logger } from 'winston';
+import { PassThrough } from 'stream';
+import { Config } from '@backstage/config';
 
 import {
   GeneratorBase,
@@ -26,18 +28,39 @@ import {
 } from './types';
 import { runDockerContainer, runCommand } from './helpers';
 
-const commandExists = require('command-exists-promise');
+type TechdocsGeneratorOptions = {
+  // This option enables users to configure if they want to use TechDocs container
+  // or generate without the container.
+  // This is used to avoid running into Docker in Docker environment.
+  runGeneratorIn: string;
+};
+
+const createStream = (): [string[], PassThrough] => {
+  const log = [] as Array<string>;
+
+  const stream = new PassThrough();
+  stream.on('data', chunk => {
+    const textValue = chunk.toString().trim();
+    if (textValue?.length > 1) log.push(textValue);
+  });
+
+  return [log, stream];
+};
 
 export class TechdocsGenerator implements GeneratorBase {
   private readonly logger: Logger;
+  private readonly options: TechdocsGeneratorOptions;
 
-  constructor(logger: Logger) {
+  constructor(logger: Logger, config: Config) {
     this.logger = logger;
+    this.options = {
+      runGeneratorIn:
+        config.getOptionalString('techdocs.generators.techdocs') ?? 'docker',
+    };
   }
 
   public async run({
     directory,
-    logStream,
     dockerClient,
   }: GeneratorRunOptions): Promise<GeneratorRunResult> {
     const tmpdirPath = os.tmpdir();
@@ -46,35 +69,46 @@ export class TechdocsGenerator implements GeneratorBase {
     const resultDir = fs.mkdtempSync(
       path.join(tmpdirResolvedPath, 'techdocs-tmp-'),
     );
+    const [log, logStream] = createStream();
 
     try {
-      const mkdocsInstalled = await commandExists('mkdocs');
-      if (mkdocsInstalled) {
-        await runCommand({
-          command: 'mkdocs',
-          args: ['build', '-d', resultDir, '-v'],
-          options: {
-            cwd: directory,
-          },
-          logStream,
-        });
-      } else {
-        await runDockerContainer({
-          imageName: 'spotify/techdocs',
-          args: ['build', '-d', '/result'],
-          logStream,
-          docsDir: directory,
-          resultDir,
-          dockerClient,
-        });
-        this.logger.info(
-          `[TechDocs]: Successfully generated docs from ${directory} into ${resultDir}`,
-        );
+      switch (this.options.runGeneratorIn) {
+        case 'local':
+          await runCommand({
+            command: 'mkdocs',
+            args: ['build', '-d', resultDir, '-v'],
+            options: {
+              cwd: directory,
+            },
+            logStream,
+          });
+          this.logger.info(
+            `[TechDocs]: Successfully generated docs from ${directory} into ${resultDir} using local mkdocs`,
+          );
+          break;
+        case 'docker':
+          await runDockerContainer({
+            imageName: 'spotify/techdocs',
+            args: ['build', '-d', '/result'],
+            logStream,
+            docsDir: directory,
+            resultDir,
+            dockerClient,
+          });
+          this.logger.info(
+            `[TechDocs]: Successfully generated docs from ${directory} into ${resultDir} using techdocs-container`,
+          );
+          break;
+        default:
+          throw new Error(
+            `Invalid config value "${this.options.runGeneratorIn}" provided in 'techdocs.generators.techdocs'.`,
+          );
       }
     } catch (error) {
       this.logger.debug(
         `[TechDocs]: Failed to generate docs from ${directory} into ${resultDir}`,
       );
+      this.logger.debug(`[TechDocs]: Build failed with error: ${log}`);
       throw new Error(
         `Failed to generate docs from ${directory} into ${resultDir} with error ${error.message}`,
       );