From 130d0842a1d4f62a5aba2358eed5cbf2f5d8a953 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Thu, 2 May 2024 14:22:20 +0200 Subject: [PATCH] techdocs-backend: instantiate containerRunner internally Signed-off-by: Vincenzo Scamporlino --- .../backend-legacy/src/plugins/techdocs.ts | 7 - .../src/commands/generate/generate.ts | 14 -- plugins/techdocs-backend/src/plugin.ts | 7 - plugins/techdocs-node/package.json | 1 + .../stages/generate/DockerContainerRunner.ts | 137 ++++++++++++++++++ .../src/stages/generate/generators.ts | 2 - .../src/stages/generate/index.ts | 2 + .../src/stages/generate/techdocs.ts | 10 +- .../src/stages/generate/types.ts | 30 +++- yarn.lock | 1 + 10 files changed, 174 insertions(+), 37 deletions(-) create mode 100644 plugins/techdocs-node/src/stages/generate/DockerContainerRunner.ts diff --git a/packages/backend-legacy/src/plugins/techdocs.ts b/packages/backend-legacy/src/plugins/techdocs.ts index 6e369f8f0f..21048d4bb9 100644 --- a/packages/backend-legacy/src/plugins/techdocs.ts +++ b/packages/backend-legacy/src/plugins/techdocs.ts @@ -14,14 +14,12 @@ * limitations under the License. */ -import { DockerContainerRunner } from '@backstage/backend-common'; import { createRouter, Generators, Preparers, Publisher, } from '@backstage/plugin-techdocs-backend'; -import Docker from 'dockerode'; import { Router } from 'express'; import { PluginEnvironment } from '../types'; @@ -34,14 +32,9 @@ export default async function createPlugin( reader: env.reader, }); - // Docker client (conditionally) used by the generators, based on techdocs.generators config. - const dockerClient = new Docker(); - const containerRunner = new DockerContainerRunner({ dockerClient }); - // Generators are used for generating documentation sites. const generators = await Generators.fromConfig(env.config, { logger: env.logger, - containerRunner, }); // Publisher is used for diff --git a/packages/techdocs-cli/src/commands/generate/generate.ts b/packages/techdocs-cli/src/commands/generate/generate.ts index bf6910ff4a..4553c18810 100644 --- a/packages/techdocs-cli/src/commands/generate/generate.ts +++ b/packages/techdocs-cli/src/commands/generate/generate.ts @@ -17,16 +17,11 @@ import { resolve } from 'path'; import { OptionValues } from 'commander'; import fs from 'fs-extra'; -import Docker from 'dockerode'; import { TechdocsGenerator, ParsedLocationAnnotation, getMkdocsYml, } from '@backstage/plugin-techdocs-node'; -import { - ContainerRunner, - DockerContainerRunner, -} from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { convertTechDocsRefToLocationAnnotation, @@ -75,14 +70,6 @@ export default async function generate(opts: OptionValues) { }, }); - // Docker client (conditionally) used by the generators, based on techdocs.generators config. - let containerRunner: ContainerRunner | undefined; - - if (opts.docker) { - const dockerClient = new Docker(); - containerRunner = new DockerContainerRunner({ dockerClient }); - } - let parsedLocationAnnotation = {} as ParsedLocationAnnotation; if (opts.techdocsRef) { try { @@ -97,7 +84,6 @@ export default async function generate(opts: OptionValues) { // Generate docs using @backstage/plugin-techdocs-node const techdocsGenerator = await TechdocsGenerator.fromConfig(config, { logger, - containerRunner, }); logger.info('Generating documentation...'); diff --git a/plugins/techdocs-backend/src/plugin.ts b/plugins/techdocs-backend/src/plugin.ts index 094335a0f0..e3a0c60a15 100644 --- a/plugins/techdocs-backend/src/plugin.ts +++ b/plugins/techdocs-backend/src/plugin.ts @@ -16,7 +16,6 @@ import { cacheToPluginCacheManager, - DockerContainerRunner, loggerToWinstonLogger, } from '@backstage/backend-common'; import { @@ -36,7 +35,6 @@ import { techdocsGeneratorExtensionPoint, techdocsPreparerExtensionPoint, } from '@backstage/plugin-techdocs-node'; -import Docker from 'dockerode'; import { createRouter } from '@backstage/plugin-techdocs-backend'; import * as winston from 'winston'; @@ -118,14 +116,9 @@ export const techdocsPlugin = createBackendPlugin({ preparers.register(protocol, preparer); } - // Docker client (conditionally) used by the generators, based on techdocs.generators config. - const dockerClient = new Docker(); - const containerRunner = new DockerContainerRunner({ dockerClient }); - // Generators are used for generating documentation sites. const generators = await Generators.fromConfig(config, { logger: winstonLogger, - containerRunner, customGenerator: customTechdocsGenerator, }); diff --git a/plugins/techdocs-node/package.json b/plugins/techdocs-node/package.json index 7ef38c56ff..8b7f87b230 100644 --- a/plugins/techdocs-node/package.json +++ b/plugins/techdocs-node/package.json @@ -64,6 +64,7 @@ "@smithy/node-http-handler": "^2.1.7", "@trendyol-js/openstack-swift-sdk": "^0.0.7", "@types/express": "^4.17.6", + "dockerode": "^4.0.0", "express": "^4.17.1", "fs-extra": "^11.2.0", "git-url-parse": "^14.0.0", diff --git a/plugins/techdocs-node/src/stages/generate/DockerContainerRunner.ts b/plugins/techdocs-node/src/stages/generate/DockerContainerRunner.ts new file mode 100644 index 0000000000..31cb6bcaa8 --- /dev/null +++ b/plugins/techdocs-node/src/stages/generate/DockerContainerRunner.ts @@ -0,0 +1,137 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Docker from 'dockerode'; +import fs from 'fs-extra'; +import { ForwardedError } from '@backstage/errors'; +import { PassThrough } from 'stream'; +import { pipeline as pipelineStream } from 'stream'; +import { promisify } from 'util'; +import { ContainerRunner, RunContainerOptions } from './types'; + +const pipeline = promisify(pipelineStream); + +export type UserOptions = { + User?: string; +}; + +/** + * A {@link ContainerRunner} for Docker containers. + * + * @public + */ +export class DockerContainerRunner implements ContainerRunner { + private readonly dockerClient: Docker; + + constructor() { + this.dockerClient = new Docker(); + } + + async runContainer(options: RunContainerOptions) { + const { + imageName, + command, + args, + logStream = new PassThrough(), + mountDirs = {}, + workingDir, + envVars = {}, + pullImage = true, + defaultUser = false, + } = options; + + // Show a better error message when Docker is unavailable. + try { + await this.dockerClient.ping(); + } catch (e) { + throw new ForwardedError( + 'This operation requires Docker. Docker does not appear to be available. Docker.ping() failed with', + e, + ); + } + + if (pullImage) { + await new Promise((resolve, reject) => { + this.dockerClient.pull(imageName, {}, (err, stream) => { + if (err) { + reject(err); + return; + } + + pipeline(stream, logStream, { end: false }) + .then(resolve) + .catch(reject); + }); + }); + } + + const userOptions: UserOptions = {}; + if (!defaultUser && process.getuid && process.getgid) { + // Files that are created inside the Docker container will be owned by + // root on the host system on non Mac systems, because of reasons. Mainly the fact that + // volume sharing is done using NFS on Mac and actual mounts in Linux world. + // So we set the user in the container as the same user and group id as the host. + // On Windows we don't have process.getuid nor process.getgid + userOptions.User = `${process.getuid()}:${process.getgid()}`; + } + + // Initialize volumes to mount based on mountDirs map + const Volumes: { [T: string]: object } = {}; + for (const containerDir of Object.values(mountDirs)) { + Volumes[containerDir] = {}; + } + + // Create bind volumes + const Binds: string[] = []; + for (const [hostDir, containerDir] of Object.entries(mountDirs)) { + // Need to use realpath here as Docker mounting does not like + // symlinks for binding volumes + const realHostDir = await fs.realpath(hostDir); + Binds.push(`${realHostDir}:${containerDir}`); + } + + // Create docker environment variables array + const Env = []; + for (const [key, value] of Object.entries(envVars)) { + Env.push(`${key}=${value}`); + } + + const [{ Error: error, StatusCode: statusCode }] = + await this.dockerClient.run(imageName, args, logStream, { + Volumes, + HostConfig: { + AutoRemove: true, + Binds, + }, + ...(workingDir ? { WorkingDir: workingDir } : {}), + Entrypoint: command, + Env, + ...userOptions, + } as Docker.ContainerCreateOptions); + + if (error) { + throw new Error( + `Docker failed to run with the following error message: ${error}`, + ); + } + + if (statusCode !== 0) { + throw new Error( + `Docker container returned a non-zero exit code (${statusCode})`, + ); + } + } +} diff --git a/plugins/techdocs-node/src/stages/generate/generators.ts b/plugins/techdocs-node/src/stages/generate/generators.ts index 603c714370..60798cf9c0 100644 --- a/plugins/techdocs-node/src/stages/generate/generators.ts +++ b/plugins/techdocs-node/src/stages/generate/generators.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { ContainerRunner } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { Logger } from 'winston'; @@ -42,7 +41,6 @@ export class Generators implements GeneratorBuilder { config: Config, options: { logger: Logger; - containerRunner: ContainerRunner; customGenerator?: TechdocsGenerator; }, ): Promise { diff --git a/plugins/techdocs-node/src/stages/generate/index.ts b/plugins/techdocs-node/src/stages/generate/index.ts index aaf27026c2..eb0aacff88 100644 --- a/plugins/techdocs-node/src/stages/generate/index.ts +++ b/plugins/techdocs-node/src/stages/generate/index.ts @@ -17,6 +17,8 @@ export { TechdocsGenerator } from './techdocs'; export { Generators } from './generators'; export { getMkdocsYml } from './helpers'; export type { + RunContainerOptions, + ContainerRunner, GeneratorBase, GeneratorOptions, GeneratorBuilder, diff --git a/plugins/techdocs-node/src/stages/generate/techdocs.ts b/plugins/techdocs-node/src/stages/generate/techdocs.ts index c5130a3f8f..6d7e3030a3 100644 --- a/plugins/techdocs-node/src/stages/generate/techdocs.ts +++ b/plugins/techdocs-node/src/stages/generate/techdocs.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { ContainerRunner } from '@backstage/backend-common'; import { Config } from '@backstage/config'; import path from 'path'; import { Logger } from 'winston'; @@ -36,6 +35,7 @@ import { patchMkdocsYmlWithPlugins, } from './mkdocsPatchers'; import { + ContainerRunner, GeneratorBase, GeneratorConfig, GeneratorOptions, @@ -43,6 +43,7 @@ import { GeneratorRunOptions, } from './types'; import { ForwardedError } from '@backstage/errors'; +import { DockerContainerRunner } from './DockerContainerRunner'; /** * Generates documentation files @@ -55,10 +56,9 @@ export class TechdocsGenerator implements GeneratorBase { */ public static readonly defaultDockerImage = 'spotify/techdocs:v1.2.3'; private readonly logger: Logger; - private readonly containerRunner?: ContainerRunner; private readonly options: GeneratorConfig; private readonly scmIntegrations: ScmIntegrationRegistry; - + private containerRunner?: ContainerRunner; /** * Returns a instance of TechDocs generator * @param config - A Backstage configuration @@ -157,9 +157,7 @@ export class TechdocsGenerator implements GeneratorBase { break; case 'docker': if (this.containerRunner === undefined) { - throw new Error( - "Invalid state: containerRunner cannot be undefined when runIn is 'docker'", - ); + this.containerRunner = new DockerContainerRunner(); } await this.containerRunner.runContainer({ imageName: diff --git a/plugins/techdocs-node/src/stages/generate/types.ts b/plugins/techdocs-node/src/stages/generate/types.ts index 149c3c2195..bde764c713 100644 --- a/plugins/techdocs-node/src/stages/generate/types.ts +++ b/plugins/techdocs-node/src/stages/generate/types.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { ContainerRunner } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; import { Writable } from 'stream'; import { Logger } from 'winston'; @@ -99,3 +98,32 @@ export type DefaultMkdocsContent = { docs_dir: string; plugins: String[]; }; + +/** + * Options passed to the {@link ContainerRunner.runContainer} method. + * + * @public + */ +export type RunContainerOptions = { + imageName: string; + command?: string | string[]; + args: string[]; + logStream?: Writable; + mountDirs?: Record; + workingDir?: string; + envVars?: Record; + pullImage?: boolean; + defaultUser?: boolean; +}; + +/** + * Handles the running of containers, on behalf of others. + * + * @public + */ +export interface ContainerRunner { + /** + * Runs a container image to completion. + */ + runContainer(opts: RunContainerOptions): Promise; +} diff --git a/yarn.lock b/yarn.lock index 7351a9dd46..8da874b506 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7802,6 +7802,7 @@ __metadata: "@types/recursive-readdir": ^2.2.0 "@types/supertest": ^2.0.8 aws-sdk-client-mock: ^4.0.0 + dockerode: ^4.0.0 express: ^4.17.1 fs-extra: ^11.2.0 git-url-parse: ^14.0.0