techdocs-backend: instantiate containerRunner internally

Signed-off-by: Vincenzo Scamporlino <vincenzos@spotify.com>

packages/backend-legacy/src/plugins/techdocs.ts

@@ -14,14 +14,12 @@
  * limitations under the License.
  */
 
-import { DockerContainerRunner } from '@backstage/backend-common';
 import {
   createRouter,
   Generators,
   Preparers,
   Publisher,
 } from '@backstage/plugin-techdocs-backend';
-import Docker from 'dockerode';
 import { Router } from 'express';
 import { PluginEnvironment } from '../types';
 
@@ -34,14 +32,9 @@ export default async function createPlugin(
     reader: env.reader,
   });
 
-  // Docker client (conditionally) used by the generators, based on techdocs.generators config.
-  const dockerClient = new Docker();
-  const containerRunner = new DockerContainerRunner({ dockerClient });
-
   // Generators are used for generating documentation sites.
   const generators = await Generators.fromConfig(env.config, {
     logger: env.logger,
-    containerRunner,
   });
 
   // Publisher is used for

packages/techdocs-cli/src/commands/generate/generate.ts

@@ -17,16 +17,11 @@
 import { resolve } from 'path';
 import { OptionValues } from 'commander';
 import fs from 'fs-extra';
-import Docker from 'dockerode';
 import {
   TechdocsGenerator,
   ParsedLocationAnnotation,
   getMkdocsYml,
 } from '@backstage/plugin-techdocs-node';
-import {
-  ContainerRunner,
-  DockerContainerRunner,
-} from '@backstage/backend-common';
 import { ConfigReader } from '@backstage/config';
 import {
   convertTechDocsRefToLocationAnnotation,
@@ -75,14 +70,6 @@ export default async function generate(opts: OptionValues) {
     },
   });
 
-  // Docker client (conditionally) used by the generators, based on techdocs.generators config.
-  let containerRunner: ContainerRunner | undefined;
-
-  if (opts.docker) {
-    const dockerClient = new Docker();
-    containerRunner = new DockerContainerRunner({ dockerClient });
-  }
-
   let parsedLocationAnnotation = {} as ParsedLocationAnnotation;
   if (opts.techdocsRef) {
     try {
@@ -97,7 +84,6 @@ export default async function generate(opts: OptionValues) {
   // Generate docs using @backstage/plugin-techdocs-node
   const techdocsGenerator = await TechdocsGenerator.fromConfig(config, {
     logger,
-    containerRunner,
   });
 
   logger.info('Generating documentation...');

plugins/techdocs-backend/src/plugin.ts

@@ -16,7 +16,6 @@
 
 import {
   cacheToPluginCacheManager,
-  DockerContainerRunner,
   loggerToWinstonLogger,
 } from '@backstage/backend-common';
 import {
@@ -36,7 +35,6 @@ import {
   techdocsGeneratorExtensionPoint,
   techdocsPreparerExtensionPoint,
 } from '@backstage/plugin-techdocs-node';
-import Docker from 'dockerode';
 import { createRouter } from '@backstage/plugin-techdocs-backend';
 import * as winston from 'winston';
 
@@ -118,14 +116,9 @@ export const techdocsPlugin = createBackendPlugin({
           preparers.register(protocol, preparer);
         }
 
-        // Docker client (conditionally) used by the generators, based on techdocs.generators config.
-        const dockerClient = new Docker();
-        const containerRunner = new DockerContainerRunner({ dockerClient });
-
         // Generators are used for generating documentation sites.
         const generators = await Generators.fromConfig(config, {
           logger: winstonLogger,
-          containerRunner,
           customGenerator: customTechdocsGenerator,
         });
 

plugins/techdocs-node/package.json

@@ -64,6 +64,7 @@
     "@smithy/node-http-handler": "^2.1.7",
     "@trendyol-js/openstack-swift-sdk": "^0.0.7",
     "@types/express": "^4.17.6",
+    "dockerode": "^4.0.0",
     "express": "^4.17.1",
     "fs-extra": "^11.2.0",
     "git-url-parse": "^14.0.0",

plugins/techdocs-node/src/stages/generate/DockerContainerRunner.ts

@@ -0,0 +1,137 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Docker from 'dockerode';
+import fs from 'fs-extra';
+import { ForwardedError } from '@backstage/errors';
+import { PassThrough } from 'stream';
+import { pipeline as pipelineStream } from 'stream';
+import { promisify } from 'util';
+import { ContainerRunner, RunContainerOptions } from './types';
+
+const pipeline = promisify(pipelineStream);
+
+export type UserOptions = {
+  User?: string;
+};
+
+/**
+ * A {@link ContainerRunner} for Docker containers.
+ *
+ * @public
+ */
+export class DockerContainerRunner implements ContainerRunner {
+  private readonly dockerClient: Docker;
+
+  constructor() {
+    this.dockerClient = new Docker();
+  }
+
+  async runContainer(options: RunContainerOptions) {
+    const {
+      imageName,
+      command,
+      args,
+      logStream = new PassThrough(),
+      mountDirs = {},
+      workingDir,
+      envVars = {},
+      pullImage = true,
+      defaultUser = false,
+    } = options;
+
+    // Show a better error message when Docker is unavailable.
+    try {
+      await this.dockerClient.ping();
+    } catch (e) {
+      throw new ForwardedError(
+        'This operation requires Docker. Docker does not appear to be available. Docker.ping() failed with',
+        e,
+      );
+    }
+
+    if (pullImage) {
+      await new Promise<void>((resolve, reject) => {
+        this.dockerClient.pull(imageName, {}, (err, stream) => {
+          if (err) {
+            reject(err);
+            return;
+          }
+
+          pipeline(stream, logStream, { end: false })
+            .then(resolve)
+            .catch(reject);
+        });
+      });
+    }
+
+    const userOptions: UserOptions = {};
+    if (!defaultUser && process.getuid && process.getgid) {
+      // Files that are created inside the Docker container will be owned by
+      // root on the host system on non Mac systems, because of reasons. Mainly the fact that
+      // volume sharing is done using NFS on Mac and actual mounts in Linux world.
+      // So we set the user in the container as the same user and group id as the host.
+      // On Windows we don't have process.getuid nor process.getgid
+      userOptions.User = `${process.getuid()}:${process.getgid()}`;
+    }
+
+    // Initialize volumes to mount based on mountDirs map
+    const Volumes: { [T: string]: object } = {};
+    for (const containerDir of Object.values(mountDirs)) {
+      Volumes[containerDir] = {};
+    }
+
+    // Create bind volumes
+    const Binds: string[] = [];
+    for (const [hostDir, containerDir] of Object.entries(mountDirs)) {
+      // Need to use realpath here as Docker mounting does not like
+      // symlinks for binding volumes
+      const realHostDir = await fs.realpath(hostDir);
+      Binds.push(`${realHostDir}:${containerDir}`);
+    }
+
+    // Create docker environment variables array
+    const Env = [];
+    for (const [key, value] of Object.entries(envVars)) {
+      Env.push(`${key}=${value}`);
+    }
+
+    const [{ Error: error, StatusCode: statusCode }] =
+      await this.dockerClient.run(imageName, args, logStream, {
+        Volumes,
+        HostConfig: {
+          AutoRemove: true,
+          Binds,
+        },
+        ...(workingDir ? { WorkingDir: workingDir } : {}),
+        Entrypoint: command,
+        Env,
+        ...userOptions,
+      } as Docker.ContainerCreateOptions);
+
+    if (error) {
+      throw new Error(
+        `Docker failed to run with the following error message: ${error}`,
+      );
+    }
+
+    if (statusCode !== 0) {
+      throw new Error(
+        `Docker container returned a non-zero exit code (${statusCode})`,
+      );
+    }
+  }
+}

plugins/techdocs-node/src/stages/generate/generators.ts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-import { ContainerRunner } from '@backstage/backend-common';
 import { Entity } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import { Logger } from 'winston';
@@ -42,7 +41,6 @@ export class Generators implements GeneratorBuilder {
     config: Config,
     options: {
       logger: Logger;
-      containerRunner: ContainerRunner;
       customGenerator?: TechdocsGenerator;
     },
   ): Promise<GeneratorBuilder> {

plugins/techdocs-node/src/stages/generate/index.ts

@@ -17,6 +17,8 @@ export { TechdocsGenerator } from './techdocs';
 export { Generators } from './generators';
 export { getMkdocsYml } from './helpers';
 export type {
+  RunContainerOptions,
+  ContainerRunner,
   GeneratorBase,
   GeneratorOptions,
   GeneratorBuilder,

plugins/techdocs-node/src/stages/generate/techdocs.ts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-import { ContainerRunner } from '@backstage/backend-common';
 import { Config } from '@backstage/config';
 import path from 'path';
 import { Logger } from 'winston';
@@ -36,6 +35,7 @@ import {
   patchMkdocsYmlWithPlugins,
 } from './mkdocsPatchers';
 import {
+  ContainerRunner,
   GeneratorBase,
   GeneratorConfig,
   GeneratorOptions,
@@ -43,6 +43,7 @@ import {
   GeneratorRunOptions,
 } from './types';
 import { ForwardedError } from '@backstage/errors';
+import { DockerContainerRunner } from './DockerContainerRunner';
 
 /**
  * Generates documentation files
@@ -55,10 +56,9 @@ export class TechdocsGenerator implements GeneratorBase {
    */
   public static readonly defaultDockerImage = 'spotify/techdocs:v1.2.3';
   private readonly logger: Logger;
-  private readonly containerRunner?: ContainerRunner;
   private readonly options: GeneratorConfig;
   private readonly scmIntegrations: ScmIntegrationRegistry;
-
+  private containerRunner?: ContainerRunner;
   /**
    * Returns a instance of TechDocs generator
    * @param config - A Backstage configuration
@@ -157,9 +157,7 @@ export class TechdocsGenerator implements GeneratorBase {
           break;
         case 'docker':
           if (this.containerRunner === undefined) {
-            throw new Error(
-              "Invalid state: containerRunner cannot be undefined when runIn is 'docker'",
-            );
+            this.containerRunner = new DockerContainerRunner();
           }
           await this.containerRunner.runContainer({
             imageName:

plugins/techdocs-node/src/stages/generate/types.ts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-import { ContainerRunner } from '@backstage/backend-common';
 import { Entity } from '@backstage/catalog-model';
 import { Writable } from 'stream';
 import { Logger } from 'winston';
@@ -99,3 +98,32 @@ export type DefaultMkdocsContent = {
   docs_dir: string;
   plugins: String[];
 };
+
+/**
+ * Options passed to the {@link ContainerRunner.runContainer} method.
+ *
+ * @public
+ */
+export type RunContainerOptions = {
+  imageName: string;
+  command?: string | string[];
+  args: string[];
+  logStream?: Writable;
+  mountDirs?: Record<string, string>;
+  workingDir?: string;
+  envVars?: Record<string, string>;
+  pullImage?: boolean;
+  defaultUser?: boolean;
+};
+
+/**
+ * Handles the running of containers, on behalf of others.
+ *
+ * @public
+ */
+export interface ContainerRunner {
+  /**
+   * Runs a container image to completion.
+   */
+  runContainer(opts: RunContainerOptions): Promise<void>;
+}

yarn.lock

@@ -7802,6 +7802,7 @@ __metadata:
     "@types/recursive-readdir": ^2.2.0
     "@types/supertest": ^2.0.8
     aws-sdk-client-mock: ^4.0.0
+    dockerode: ^4.0.0
     express: ^4.17.1
     fs-extra: ^11.2.0
     git-url-parse: ^14.0.0