From e9b7db7b29b1e8fa4fba471f3f6de75fce842634 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Mon, 1 Mar 2021 17:33:48 +0100 Subject: [PATCH] backend-common: Add runDockerContainer shared by Scaffolder and TechDocs Signed-off-by: Himanshu Mishra --- packages/backend-common/package.json | 2 + packages/backend-common/src/index.ts | 7 +- .../backend-common/src/util/docker.test.ts | 200 ++++++++++++++++++ packages/backend-common/src/util/docker.ts | 119 +++++++++++ packages/backend-common/src/util/index.ts | 17 ++ .../src/stages/generate/helpers.test.ts | 133 ------------ .../src/stages/generate/helpers.ts | 88 -------- .../src/stages/generate/techdocs.ts | 6 +- .../stages/templater/cookiecutter.ts | 15 +- .../scaffolder/stages/templater/cra/index.ts | 8 +- .../stages/templater/helpers.test.ts | 184 ---------------- .../scaffolder/stages/templater/helpers.ts | 97 +-------- 12 files changed, 359 insertions(+), 517 deletions(-) create mode 100644 packages/backend-common/src/util/docker.test.ts create mode 100644 packages/backend-common/src/util/docker.ts create mode 100644 packages/backend-common/src/util/index.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 3cd817427f..4205e12715 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -35,12 +35,14 @@ "@backstage/integration": "^0.5.0", "@octokit/rest": "^18.0.12", "@types/cors": "^2.8.6", + "@types/dockerode": "^3.2.1", "@types/express": "^4.17.6", "archiver": "^5.0.2", "compression": "^1.7.4", "concat-stream": "^2.0.0", "cors": "^2.8.5", "cross-fetch": "^3.0.6", + "dockerode": "^3.2.1", "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.1", diff --git a/packages/backend-common/src/index.ts b/packages/backend-common/src/index.ts index 3f04f32b34..f3653f2627 100644 --- a/packages/backend-common/src/index.ts +++ b/packages/backend-common/src/index.ts @@ -18,10 +18,11 @@ export * from './config'; export * from './database'; export * from './discovery'; export * from './errors'; +export * from './hot'; export * from './logging'; export * from './middleware'; -export * from './reading'; -export * from './service'; export * from './paths'; -export * from './hot'; +export * from './reading'; export * from './scm'; +export * from './service'; +export * from './util'; diff --git a/packages/backend-common/src/util/docker.test.ts b/packages/backend-common/src/util/docker.test.ts new file mode 100644 index 0000000000..7f9bb2da33 --- /dev/null +++ b/packages/backend-common/src/util/docker.test.ts @@ -0,0 +1,200 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Docker from 'dockerode'; +import fs from 'fs'; +import os from 'os'; +import Stream, { PassThrough } from 'stream'; +import { runDockerContainer, UserOptions } from './docker'; + +const mockDocker = new Docker() as jest.Mocked; + +describe('runDockerContainer', () => { + beforeEach(() => { + jest.spyOn(mockDocker, 'pull').mockImplementation((async ( + _image: string, + _something: any, + handler: (err: Error | undefined, stream: PassThrough) => void, + ) => { + const mockStream = new PassThrough(); + handler(undefined, mockStream); + mockStream.end(); + }) as any); + + jest + .spyOn(mockDocker, 'run') + .mockResolvedValue([{ Error: null, StatusCode: 0 }]); + + jest + .spyOn(mockDocker, 'ping') + .mockResolvedValue(Buffer.from('OK', 'utf-8')); + }); + + const imageName = 'dockerOrg/image'; + const args = ['bash', '-c', 'echo test']; + const inputDir = os.tmpdir(); + const outputDir = os.tmpdir(); + + it('should pull the docker container', async () => { + await runDockerContainer({ + imageName, + args, + inputDir, + outputDir, + dockerClient: mockDocker, + }); + + expect(mockDocker.pull).toHaveBeenCalledWith( + imageName, + {}, + expect.any(Function), + ); + }); + + it('should call the dockerClient run command with the correct arguments passed through', async () => { + await runDockerContainer({ + imageName, + args, + inputDir, + outputDir, + dockerClient: mockDocker, + }); + + expect(mockDocker.run).toHaveBeenCalledWith( + imageName, + args, + expect.any(Stream), + expect.objectContaining({ + HostConfig: { + Binds: expect.arrayContaining([ + `${await fs.promises.realpath(inputDir)}:/input`, + `${await fs.promises.realpath(outputDir)}:/output`, + ]), + }, + Volumes: { + '/input': {}, + '/output': {}, + }, + }), + ); + }); + + it('should ping docker to test availability', async () => { + await runDockerContainer({ + imageName, + args, + inputDir, + outputDir, + dockerClient: mockDocker, + }); + + expect(mockDocker.ping).toHaveBeenCalled(); + }); + + it('should pass through the user and group id from the host machine and set the home dir', async () => { + await runDockerContainer({ + imageName, + args, + inputDir, + outputDir, + dockerClient: mockDocker, + }); + + const userOptions: UserOptions = {}; + if (process.getuid && process.getgid) { + userOptions.User = `${process.getuid()}:${process.getgid()}`; + } + + expect(mockDocker.run).toHaveBeenCalledWith( + imageName, + args, + expect.any(Stream), + expect.objectContaining({ + ...userOptions, + }), + ); + }); + + it('throws a correct error if the command fails in docker', async () => { + mockDocker.run.mockResolvedValueOnce([ + { + Error: new Error('Something went wrong with docker'), + StatusCode: 0, + }, + ]); + + await expect( + runDockerContainer({ + imageName, + args, + inputDir, + outputDir, + dockerClient: mockDocker, + }), + ).rejects.toThrow(/Something went wrong with docker/); + }); + + describe('where docker is unavailable', () => { + const dockerError = 'a docker error'; + + beforeEach(() => { + jest.spyOn(mockDocker, 'ping').mockImplementationOnce(() => { + throw new Error(dockerError); + }); + }); + + it('should throw with a descriptive error message including the docker error message', async () => { + await expect( + runDockerContainer({ + imageName, + args, + inputDir, + outputDir, + dockerClient: mockDocker, + }), + ).rejects.toThrow(new RegExp(`.+: ${dockerError}`)); + }); + }); + + it('should pass through the log stream to the docker client', async () => { + const logStream = new PassThrough(); + await runDockerContainer({ + imageName, + args, + inputDir, + outputDir, + logStream, + dockerClient: mockDocker, + }); + + expect(mockDocker.run).toHaveBeenCalledWith( + imageName, + args, + logStream, + expect.objectContaining({ + HostConfig: { + Binds: expect.arrayContaining([ + `${await fs.promises.realpath(inputDir)}:/input`, + `${await fs.promises.realpath(outputDir)}:/output`, + ]), + }, + Volumes: { + '/input': {}, + '/output': {}, + }, + }), + ); + }); +}); diff --git a/packages/backend-common/src/util/docker.ts b/packages/backend-common/src/util/docker.ts new file mode 100644 index 0000000000..2b73915b0c --- /dev/null +++ b/packages/backend-common/src/util/docker.ts @@ -0,0 +1,119 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Docker from 'dockerode'; +import fs from 'fs'; +import { PassThrough, Writable } from 'stream'; + +export type UserOptions = { + User?: string; +}; + +export type RunDockerContainerOptions = { + imageName: string; + args: string[]; + logStream?: Writable; + inputDir: string; + outputDir: string; + dockerClient: Docker; + createOptions?: Docker.ContainerCreateOptions; +}; + +/** + * + * @param options the options object + * @param options.imageName the image to run + * @param options.args the arguments to pass the container + * @param options.logStream the log streamer to capture log messages + * @param options.inputDir the /input path inside the container + * @param options.outputDir the /output path inside the container + * @param options.dockerClient the dockerClient to use + */ +export const runDockerContainer = async ({ + imageName, + args, + logStream = new PassThrough(), + inputDir, + outputDir, + dockerClient, + createOptions = {}, +}: RunDockerContainerOptions) => { + // Show a better error message when Docker is unavailable. + try { + await dockerClient.ping(); + } catch (e) { + throw new Error( + `This operation requires Docker. Docker does not appear to be available. Docker.ping() failed with: ${e.message}`, + ); + } + + await new Promise((resolve, reject) => { + dockerClient.pull(imageName, {}, (err, stream) => { + if (err) return reject(err); + stream.pipe(logStream, { end: false }); + stream.on('end', () => resolve()); + stream.on('error', (error: Error) => reject(error)); + return undefined; + }); + }); + + const userOptions: UserOptions = {}; + // @ts-ignore + if (process.getuid && process.getgid) { + // Files that are created inside the Docker container will be owned by + // root on the host system on non Mac systems, because of reasons. Mainly the fact that + // volume sharing is done using NFS on Mac and actual mounts in Linux world. + // So we set the user in the container as the same user and group id as the host. + // On Windows we don't have process.getuid nor process.getgid + userOptions.User = `${process.getuid()}:${process.getgid()}`; + } + + const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run( + imageName, + args, + logStream, + { + Volumes: { '/output': {}, '/input': {} }, + HostConfig: { + Binds: [ + // Need to use realpath here as Docker mounting does not like + // symlinks for binding volumes + `${await fs.promises.realpath(outputDir)}:/output`, + `${await fs.promises.realpath(inputDir)}:/input`, + ], + }, + ...userOptions, + // Set the home directory inside the container as something that applications can + // write to, otherwise they will just flop and fail trying to write to / + Env: ['HOME=/tmp'], + ...createOptions, + }, + ); + + if (error) { + throw new Error( + `Docker failed to run with the following error message: ${error}`, + ); + } + + if (statusCode !== 0) { + throw new Error( + `Docker container returned a non-zero exit code (${statusCode})`, + ); + } + + return { error, statusCode }; +}; diff --git a/packages/backend-common/src/util/index.ts b/packages/backend-common/src/util/index.ts new file mode 100644 index 0000000000..04715824b4 --- /dev/null +++ b/packages/backend-common/src/util/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { runDockerContainer } from './docker'; diff --git a/packages/techdocs-common/src/stages/generate/helpers.test.ts b/packages/techdocs-common/src/stages/generate/helpers.test.ts index fbca6d1ed2..05402d0cf5 100644 --- a/packages/techdocs-common/src/stages/generate/helpers.test.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.test.ts @@ -14,12 +14,10 @@ * limitations under the License. */ import { getVoidLogger } from '@backstage/backend-common'; -import Docker from 'dockerode'; import fs from 'fs-extra'; import mockFs from 'mock-fs'; import os from 'os'; import path, { resolve as resolvePath } from 'path'; -import Stream, { PassThrough } from 'stream'; import { ParsedLocationAnnotation } from '../../helpers'; import { RemoteProtocol } from '../prepare/types'; import { @@ -28,9 +26,7 @@ import { getRepoUrlFromLocationAnnotation, isValidRepoUrlForMkdocs, patchMkdocsYmlPreBuild, - runDockerContainer, storeEtagMetadata, - UserOptions, } from './helpers'; const mockEntity = { @@ -41,8 +37,6 @@ const mockEntity = { }, }; -const mockDocker = new Docker() as jest.Mocked; - const mkdocsYml = fs.readFileSync( resolvePath(__filename, '../__fixtures__/mkdocs.yml'), ); @@ -60,133 +54,6 @@ describe('helpers', () => { }); }); - describe('runDockerContainer', () => { - beforeEach(() => { - jest.spyOn(mockDocker, 'pull').mockImplementation((async ( - _image: string, - _something: any, - handler: (err: Error | undefined, stream: PassThrough) => void, - ) => { - const mockStream = new PassThrough(); - handler(undefined, mockStream); - mockStream.end(); - }) as any); - - jest - .spyOn(mockDocker, 'run') - .mockResolvedValue([{ Error: null, StatusCode: 0 }]); - - jest - .spyOn(mockDocker, 'ping') - .mockResolvedValue(Buffer.from('OK', 'utf-8')); - }); - - const imageName = 'spotify/techdocs'; - const args = ['build', '-d', '/result']; - const docsDir = os.tmpdir(); - const outputDir = os.tmpdir(); - - it('should pull the techdocs docker container', async () => { - await runDockerContainer({ - imageName, - args, - docsDir, - outputDir, - dockerClient: mockDocker, - }); - - expect(mockDocker.pull).toHaveBeenCalledWith( - imageName, - {}, - expect.any(Function), - ); - }); - - it('should run the techdocs docker container', async () => { - await runDockerContainer({ - imageName, - args, - docsDir, - outputDir, - dockerClient: mockDocker, - }); - - expect(mockDocker.run).toHaveBeenCalledWith( - imageName, - args, - expect.any(Stream), - expect.objectContaining({ - Volumes: { - '/content': {}, - '/result': {}, - }, - WorkingDir: '/content', - HostConfig: { - Binds: [`${docsDir}:/content`, `${outputDir}:/result`], - }, - }), - ); - }); - - it('should ping docker to test availability', async () => { - await runDockerContainer({ - imageName, - args, - docsDir, - outputDir, - dockerClient: mockDocker, - }); - - expect(mockDocker.ping).toHaveBeenCalled(); - }); - - it('should pass through the user and group id from the host machine and set the home dir', async () => { - await runDockerContainer({ - imageName, - args, - docsDir, - outputDir, - dockerClient: mockDocker, - }); - - const userOptions: UserOptions = {}; - if (process.getuid && process.getgid) { - userOptions.User = `${process.getuid()}:${process.getgid()}`; - } - - expect(mockDocker.run).toHaveBeenCalledWith( - imageName, - args, - expect.any(Stream), - expect.objectContaining({ - ...userOptions, - }), - ); - }); - - describe('where docker is unavailable', () => { - const dockerError = 'a docker error'; - - beforeEach(() => { - jest.spyOn(mockDocker, 'ping').mockImplementationOnce(() => { - throw new Error(dockerError); - }); - }); - - it('should throw with a descriptive error message including the docker error message', async () => { - await expect( - runDockerContainer({ - imageName, - args, - docsDir, - outputDir, - dockerClient: mockDocker, - }), - ).rejects.toThrow(new RegExp(`.+: ${dockerError}`)); - }); - }); - }); - describe('isValidRepoUrlForMkdocs', () => { it('should return true for valid repo_url values for mkdocs', () => { const validRepoUrls = [ diff --git a/packages/techdocs-common/src/stages/generate/helpers.ts b/packages/techdocs-common/src/stages/generate/helpers.ts index a88ed13231..a99544a1f8 100644 --- a/packages/techdocs-common/src/stages/generate/helpers.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.ts @@ -16,7 +16,6 @@ import { Entity } from '@backstage/catalog-model'; import { spawn } from 'child_process'; -import Docker from 'dockerode'; import fs from 'fs-extra'; import yaml from 'js-yaml'; import { PassThrough, Writable } from 'stream'; @@ -34,16 +33,6 @@ export function getGeneratorKey(entity: Entity): SupportedGeneratorKey { return 'techdocs'; } -type RunDockerContainerOptions = { - imageName: string; - args: string[]; - logStream?: Writable; - docsDir: string; - outputDir: string; - dockerClient: Docker; - createOptions?: Docker.ContainerCreateOptions; -}; - export type RunCommandOptions = { command: string; args: string[]; @@ -51,83 +40,6 @@ export type RunCommandOptions = { logStream?: Writable; }; -export type UserOptions = { - User?: string; -}; - -// To be replaced by a runDockerContainer from backend-common -// shared between Scaffolder and TechDocs and any other plugin. -export async function runDockerContainer({ - imageName, - args, - logStream = new PassThrough(), - docsDir, - outputDir, - dockerClient, - createOptions, -}: RunDockerContainerOptions) { - try { - await dockerClient.ping(); - } catch (e) { - throw new Error( - `This operation requires Docker. Docker does not appear to be available. Docker.ping() failed with: ${e.message}`, - ); - } - - await new Promise((resolve, reject) => { - dockerClient.pull(imageName, {}, (err, stream) => { - if (err) return reject(err); - stream.pipe(logStream, { end: false }); - stream.on('end', () => resolve()); - stream.on('error', (error: Error) => reject(error)); - return undefined; - }); - }); - - const userOptions: UserOptions = {}; - // @ts-ignore - if (process.getuid && process.getgid) { - // Files that are created inside the Docker container will be owned by - // root on the host system on non Mac systems, because of reasons. Mainly the fact that - // volume sharing is done using NFS on Mac and actual mounts in Linux world. - // So we set the user in the container as the same user and group id as the host. - // On Windows we don't have process.getuid nor process.getgid - userOptions.User = `${process.getuid()}:${process.getgid()}`; - } - - const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run( - imageName, - args, - logStream, - { - Volumes: { - '/content': {}, - '/result': {}, - }, - WorkingDir: '/content', - HostConfig: { - Binds: [`${docsDir}:/content`, `${outputDir}:/result`], - }, - ...userOptions, - ...createOptions, - }, - ); - - if (error) { - throw new Error( - `Docker failed to run with the following error message: ${error}`, - ); - } - - if (statusCode !== 0) { - throw new Error( - `Docker container returned a non-zero exit code (${statusCode})`, - ); - } - - return { error, statusCode }; -} - /** * * @param options the options object diff --git a/packages/techdocs-common/src/stages/generate/techdocs.ts b/packages/techdocs-common/src/stages/generate/techdocs.ts index 133755ecd2..e25ce696f6 100644 --- a/packages/techdocs-common/src/stages/generate/techdocs.ts +++ b/packages/techdocs-common/src/stages/generate/techdocs.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { runDockerContainer } from '@backstage/backend-common'; import { Config } from '@backstage/config'; import path from 'path'; import { PassThrough } from 'stream'; @@ -22,7 +23,6 @@ import { addBuildTimestampMetadata, patchMkdocsYmlPreBuild, runCommand, - runDockerContainer, storeEtagMetadata, } from './helpers'; import { GeneratorBase, GeneratorRunOptions } from './types'; @@ -96,9 +96,9 @@ export class TechdocsGenerator implements GeneratorBase { case 'docker': await runDockerContainer({ imageName: 'spotify/techdocs', - args: ['build', '-d', '/result'], + args: ['build', '-d', '/output'], logStream, - docsDir: inputDir, + inputDir, outputDir, dockerClient, }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts index 9568af2a15..a71af1d3bf 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import fs from 'fs-extra'; +import { runDockerContainer } from '@backstage/backend-common'; import { JsonValue } from '@backstage/config'; -import { runDockerContainer, runCommand } from './helpers'; -import { TemplaterBase, TemplaterRunOptions } from '.'; +import fs from 'fs-extra'; import path from 'path'; +import { TemplaterBase, TemplaterRunOptions } from '.'; +import { runCommand } from './helpers'; const commandExists = require('command-exists-promise'); @@ -71,12 +72,12 @@ export class CookieCutter implements TemplaterBase { 'cookiecutter', '--no-input', '-o', - '/result', - '/template', + '/output', + '/input', '--verbose', ], - templateDir, - resultDir: intermediateDir, + inputDir: templateDir, + outputDir: intermediateDir, logStream, dockerClient, }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/index.ts index 5e957e0504..4f871516e9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/index.ts @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { runDockerContainer } from '@backstage/backend-common'; import fs from 'fs-extra'; -import { runDockerContainer } from '../helpers'; -import { TemplaterBase, TemplaterRunOptions } from '..'; import path from 'path'; import * as yaml from 'yaml'; +import { TemplaterBase, TemplaterRunOptions } from '..'; // TODO(blam): Replace with the universal import from github-actions after a release // As it will break the E2E without it @@ -49,8 +49,8 @@ export class CreateReactAppTemplater implements TemplaterBase { componentName as string, withTypescript ? ' --template typescript' : '', ], - templateDir: intermediateDir, - resultDir: intermediateDir, + inputDir: intermediateDir, + outputDir: intermediateDir, logStream: logStream, dockerClient: dockerClient, createOptions: { diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts deleted file mode 100644 index 57e47f4c27..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import Stream, { PassThrough } from 'stream'; -import os from 'os'; -import fs from 'fs'; -import Docker from 'dockerode'; -import { UserOptions, runDockerContainer } from './helpers'; - -describe('helpers', () => { - const mockDocker = new Docker() as jest.Mocked; - - beforeEach(() => { - jest - .spyOn(mockDocker, 'run') - .mockResolvedValue([{ Error: null, StatusCode: 0 }]); - jest.spyOn(mockDocker, 'pull').mockImplementation((async ( - _image: string, - _something: any, - handler: (err: Error | undefined, stream: PassThrough) => void, - ) => { - const mockStream = new PassThrough(); - handler(undefined, mockStream); - mockStream.end(); - }) as any); - }); - - describe('runDockerContainer', () => { - const imageName = 'blam/github:ben'; - const args = ['bash', '-c', 'echo lol']; - const templateDir = os.tmpdir(); - const resultDir = os.tmpdir(); - - it('will pull the docker container before running', async () => { - await runDockerContainer({ - imageName, - args, - templateDir, - resultDir, - dockerClient: mockDocker, - }); - - expect(mockDocker.pull).toHaveBeenCalledWith( - imageName, - {}, - expect.any(Function), - ); - }); - it('should call the dockerClient run command with the correct arguments passed through', async () => { - await runDockerContainer({ - imageName, - args, - templateDir, - resultDir, - dockerClient: mockDocker, - }); - - expect(mockDocker.run).toHaveBeenCalledWith( - imageName, - args, - expect.any(Stream), - expect.objectContaining({ - HostConfig: { - Binds: expect.arrayContaining([ - `${await fs.promises.realpath(templateDir)}:/template`, - `${await fs.promises.realpath(resultDir)}:/result`, - ]), - }, - Volumes: { - '/template': {}, - '/result': {}, - }, - }), - ); - }); - - it('throws a correct error if the templating fails in docker', async () => { - mockDocker.run.mockResolvedValueOnce([ - { - Error: new Error('Something went wrong with docker'), - StatusCode: 0, - }, - ]); - - await expect( - runDockerContainer({ - imageName, - args, - templateDir, - resultDir, - dockerClient: mockDocker, - }), - ).rejects.toThrow(/Something went wrong with docker/); - }); - - it('throws a correct error when the response code of the container is non-zero', async () => { - mockDocker.run.mockResolvedValueOnce([ - { - Error: null, - StatusCode: 123, - }, - ]); - - await expect( - runDockerContainer({ - imageName, - args, - templateDir, - resultDir, - dockerClient: mockDocker, - }), - ).rejects.toThrow( - /Docker container returned a non-zero exit code \(123\)/, - ); - }); - - it('should pass through the user and group id from the host machine and set the home dir', async () => { - await runDockerContainer({ - imageName, - args, - templateDir, - resultDir, - dockerClient: mockDocker, - }); - - const userOptions: UserOptions = {}; - if (process.getuid && process.getgid) { - userOptions.User = `${process.getuid()}:${process.getgid()}`; - } - - expect(mockDocker.run).toHaveBeenCalledWith( - imageName, - args, - expect.any(Stream), - expect.objectContaining({ - ...userOptions, - Env: ['HOME=/tmp'], - }), - ); - }); - - it('should pass through the log stream to the docker client', async () => { - const logStream = new PassThrough(); - await runDockerContainer({ - imageName, - args, - templateDir, - resultDir, - logStream, - dockerClient: mockDocker, - }); - - expect(mockDocker.run).toHaveBeenCalledWith( - imageName, - args, - logStream, - expect.objectContaining({ - HostConfig: { - Binds: expect.arrayContaining([ - `${await fs.promises.realpath(templateDir)}:/template`, - `${await fs.promises.realpath(resultDir)}:/result`, - ]), - }, - Volumes: { - '/template': {}, - '/result': {}, - }, - }), - ); - }); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts index 645133a4ae..cb1e8532f2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts @@ -13,22 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Writable, PassThrough } from 'stream'; -import Docker from 'dockerode'; -import fs from 'fs'; -import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { InputError } from '@backstage/backend-common'; +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { spawn } from 'child_process'; - -export type RunDockerContainerOptions = { - imageName: string; - args: string[]; - logStream?: Writable; - resultDir: string; - templateDir: string; - dockerClient: Docker; - createOptions?: Docker.ContainerCreateOptions; -}; +import { PassThrough, Writable } from 'stream'; export type RunCommandOptions = { command: string; @@ -36,10 +24,6 @@ export type RunCommandOptions = { logStream?: Writable; }; -export type UserOptions = { - User?: string; -}; - /** * Gets the templater key to use for templating from the entity * @param entity Template entity @@ -89,80 +73,3 @@ export const runCommand = async ({ }); }); }; - -/** - * - * @param options the options object - * @param options.imageName the image to run - * @param options.args the arguments to pass the container - * @param options.logStream the log streamer to capture log messages - * @param options.resultDir the /result path inside the container - * @param options.templateDir the /template path inside the container - * @param options.dockerClient the dockerClient to use - */ -export const runDockerContainer = async ({ - imageName, - args, - logStream = new PassThrough(), - resultDir, - templateDir, - dockerClient, - createOptions = {}, -}: RunDockerContainerOptions) => { - await new Promise((resolve, reject) => { - dockerClient.pull(imageName, {}, (err, stream) => { - if (err) return reject(err); - stream.pipe(logStream, { end: false }); - stream.on('end', () => resolve()); - stream.on('error', (error: Error) => reject(error)); - return undefined; - }); - }); - - const userOptions: UserOptions = {}; - // @ts-ignore - if (process.getuid && process.getgid) { - // Files that are created inside the Docker container will be owned by - // root on the host system on non Mac systems, because of reasons. Mainly the fact that - // volume sharing is done using NFS on Mac and actual mounts in Linux world. - // So we set the user in the container as the same user and group id as the host. - // On Windows we don't have process.getuid nor process.getgid - userOptions.User = `${process.getuid()}:${process.getgid()}`; - } - - const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run( - imageName, - args, - logStream, - { - Volumes: { '/result': {}, '/template': {} }, - HostConfig: { - Binds: [ - // Need to use realpath here as Docker mounting does not like - // symlinks for binding volumes - `${await fs.promises.realpath(resultDir)}:/result`, - `${await fs.promises.realpath(templateDir)}:/template`, - ], - }, - ...userOptions, - // Set the home directory inside the container as something that applications can - // write to, otherwise they will just flop and fail trying to write to / - Env: ['HOME=/tmp'], - ...createOptions, - }, - ); - - if (error) { - throw new Error( - `Docker failed to run with the following error message: ${error}`, - ); - } - - if (statusCode !== 0) { - throw new Error( - `Docker container returned a non-zero exit code (${statusCode})`, - ); - } - - return { error, statusCode }; -};