diff --git a/.changeset/funny-drinks-boil.md b/.changeset/funny-drinks-boil.md new file mode 100644 index 0000000000..f7b84a32a1 --- /dev/null +++ b/.changeset/funny-drinks-boil.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Add a utility function runDockerContainer used to run a docker container (currently used by Scaffolder and TechDocs for their 'generate' processes) diff --git a/.changeset/great-bees-press.md b/.changeset/great-bees-press.md new file mode 100644 index 0000000000..1582e54519 --- /dev/null +++ b/.changeset/great-bees-press.md @@ -0,0 +1,6 @@ +--- +'@backstage/techdocs-common': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Remove runDockerContainer, and start using the utility function provided by @backstage/backend-common diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index af6aa508c5..3c17a27cce 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -35,12 +35,14 @@ "@backstage/integration": "^0.5.0", "@octokit/rest": "^18.0.12", "@types/cors": "^2.8.6", + "@types/dockerode": "^3.2.1", "@types/express": "^4.17.6", "archiver": "^5.0.2", "compression": "^1.7.4", "concat-stream": "^2.0.0", "cors": "^2.8.5", "cross-fetch": "^3.0.6", + "dockerode": "^3.2.1", "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.1", diff --git a/packages/backend-common/src/index.ts b/packages/backend-common/src/index.ts index 3f04f32b34..f3653f2627 100644 --- a/packages/backend-common/src/index.ts +++ b/packages/backend-common/src/index.ts @@ -18,10 +18,11 @@ export * from './config'; export * from './database'; export * from './discovery'; export * from './errors'; +export * from './hot'; export * from './logging'; export * from './middleware'; -export * from './reading'; -export * from './service'; export * from './paths'; -export * from './hot'; +export * from './reading'; export * from './scm'; +export * from './service'; +export * from './util'; diff --git a/packages/backend-common/src/util/docker.test.ts b/packages/backend-common/src/util/docker.test.ts new file mode 100644 index 0000000000..2ec14494da --- /dev/null +++ b/packages/backend-common/src/util/docker.test.ts @@ -0,0 +1,205 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Docker from 'dockerode'; +import mockFs from 'mock-fs'; +import os from 'os'; +import path from 'path'; +import Stream, { PassThrough } from 'stream'; +import { runDockerContainer, UserOptions } from './docker'; + +const mockDocker = new Docker() as jest.Mocked; +const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; + +describe('runDockerContainer', () => { + beforeEach(() => { + mockFs({ + [rootDir]: { + input: mockFs.directory(), + output: mockFs.directory(), + }, + }); + + jest.spyOn(mockDocker, 'pull').mockImplementation((async ( + _image: string, + _something: any, + handler: (err: Error | undefined, stream: PassThrough) => void, + ) => { + const mockStream = new PassThrough(); + handler(undefined, mockStream); + mockStream.end(); + }) as any); + + jest + .spyOn(mockDocker, 'run') + .mockResolvedValue([{ Error: null, StatusCode: 0 }]); + + jest + .spyOn(mockDocker, 'ping') + .mockResolvedValue(Buffer.from('OK', 'utf-8')); + }); + + afterEach(() => { + mockFs.restore(); + }); + + const imageName = 'dockerOrg/image'; + const args = ['bash', '-c', 'echo test']; + const mountDirs = { + [path.join(rootDir, 'input')]: '/input', + [path.join(rootDir, 'output')]: '/output', + }; + const workingDir = path.join(rootDir, 'input'); + const envVars = { HOME: '/tmp', LOG_LEVEL: 'debug' }; + const envVarsArray = ['HOME=/tmp', 'LOG_LEVEL=debug']; + + it('should pull the docker container', async () => { + await runDockerContainer({ + imageName, + args, + dockerClient: mockDocker, + }); + + expect(mockDocker.pull).toHaveBeenCalledWith( + imageName, + {}, + expect.any(Function), + ); + + expect(mockDocker.run).toHaveBeenCalled(); + }); + + it('should call the dockerClient run command with the correct arguments passed through', async () => { + await runDockerContainer({ + imageName, + args, + mountDirs, + envVars, + workingDir, + dockerClient: mockDocker, + }); + + expect(mockDocker.run).toHaveBeenCalledWith( + imageName, + args, + expect.any(Stream), + expect.objectContaining({ + Env: envVarsArray, + WorkingDir: workingDir, + HostConfig: { + Binds: expect.arrayContaining([ + `${path.join(rootDir, 'input')}:/input`, + `${path.join(rootDir, 'output')}:/output`, + ]), + }, + Volumes: { + '/input': {}, + '/output': {}, + }, + }), + ); + }); + + it('should ping docker to test availability', async () => { + await runDockerContainer({ + imageName, + args, + dockerClient: mockDocker, + }); + + expect(mockDocker.ping).toHaveBeenCalled(); + }); + + it('should pass through the user and group id from the host machine and set the home dir', async () => { + await runDockerContainer({ + imageName, + args, + dockerClient: mockDocker, + }); + + const userOptions: UserOptions = {}; + if (process.getuid && process.getgid) { + userOptions.User = `${process.getuid()}:${process.getgid()}`; + } + + expect(mockDocker.run).toHaveBeenCalledWith( + imageName, + args, + expect.any(Stream), + expect.objectContaining({ + ...userOptions, + }), + ); + }); + + it('throws a correct error if the command fails in docker', async () => { + mockDocker.run.mockResolvedValueOnce([ + { + Error: new Error('Something went wrong with docker'), + StatusCode: 0, + }, + ]); + + await expect( + runDockerContainer({ + imageName, + args, + dockerClient: mockDocker, + }), + ).rejects.toThrow(/Something went wrong with docker/); + }); + + describe('where docker is unavailable', () => { + const dockerError = 'a docker error'; + + beforeEach(() => { + jest.spyOn(mockDocker, 'ping').mockImplementationOnce(() => { + throw new Error(dockerError); + }); + }); + + it('should throw with a descriptive error message including the docker error message', async () => { + await expect( + runDockerContainer({ + imageName, + args, + dockerClient: mockDocker, + }), + ).rejects.toThrow(new RegExp(`.+: ${dockerError}`)); + }); + }); + + it('should pass through the log stream to the docker client', async () => { + const logStream = new PassThrough(); + await runDockerContainer({ + imageName, + args, + logStream, + dockerClient: mockDocker, + }); + + expect(mockDocker.run).toHaveBeenCalledWith( + imageName, + args, + logStream, + expect.objectContaining({ + HostConfig: { + Binds: [], + }, + Volumes: {}, + }), + ); + }); +}); diff --git a/packages/backend-common/src/util/docker.ts b/packages/backend-common/src/util/docker.ts new file mode 100644 index 0000000000..356d645465 --- /dev/null +++ b/packages/backend-common/src/util/docker.ts @@ -0,0 +1,137 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Docker from 'dockerode'; +import fs from 'fs-extra'; +import { PassThrough, Writable } from 'stream'; + +export type UserOptions = { + User?: string; +}; + +export type RunDockerContainerOptions = { + imageName: string; + args: string[]; + logStream?: Writable; + dockerClient: Docker; + mountDirs?: Record; + workingDir?: string; + envVars?: Record; + createOptions?: Docker.ContainerCreateOptions; +}; + +/** + * + * @param options the options object + * @param options.imageName the image to run + * @param options.args the arguments to pass the container + * @param options.logStream the log streamer to capture log messages + * @param options.dockerClient the dockerClient to use + * @param options.mountDirs A map of host directories to mount on the container. + * Object Key: Path on host machine, Value: Path on Docker container + * @param options.workingDir Working dir in the container + * @param options.envVars Environment variables to set in the container. e.g. {'HOME': '/tmp'} + */ +export const runDockerContainer = async ({ + imageName, + args, + logStream = new PassThrough(), + dockerClient, + mountDirs = {}, + workingDir, + envVars = {}, + createOptions = {}, +}: RunDockerContainerOptions) => { + // Show a better error message when Docker is unavailable. + try { + await dockerClient.ping(); + } catch (e) { + throw new Error( + `This operation requires Docker. Docker does not appear to be available. Docker.ping() failed with: ${e.message}`, + ); + } + + await new Promise((resolve, reject) => { + dockerClient.pull(imageName, {}, (err, stream) => { + if (err) return reject(err); + stream.pipe(logStream, { end: false }); + stream.on('end', () => resolve()); + stream.on('error', (error: Error) => reject(error)); + return undefined; + }); + }); + + const userOptions: UserOptions = {}; + if (process.getuid && process.getgid) { + // Files that are created inside the Docker container will be owned by + // root on the host system on non Mac systems, because of reasons. Mainly the fact that + // volume sharing is done using NFS on Mac and actual mounts in Linux world. + // So we set the user in the container as the same user and group id as the host. + // On Windows we don't have process.getuid nor process.getgid + userOptions.User = `${process.getuid()}:${process.getgid()}`; + } + + // Initialize volumes to mount based on mountDirs map + const Volumes: { [T: string]: object } = {}; + for (const containerDir of Object.values(mountDirs)) { + Volumes[containerDir] = {}; + } + + // Create bind volumes + const Binds: string[] = []; + for (const [hostDir, containerDir] of Object.entries(mountDirs)) { + // Need to use realpath here as Docker mounting does not like + // symlinks for binding volumes + const realHostDir = await fs.realpath(hostDir); + Binds.push(`${realHostDir}:${containerDir}`); + } + + // Create docker environment variables array + const Env = []; + for (const [key, value] of Object.entries(envVars)) { + Env.push(`${key}=${value}`); + } + + const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run( + imageName, + args, + logStream, + { + Volumes, + HostConfig: { + Binds, + }, + ...(workingDir ? { WorkingDir: workingDir } : {}), + Env, + ...userOptions, + ...createOptions, + }, + ); + + if (error) { + throw new Error( + `Docker failed to run with the following error message: ${error}`, + ); + } + + if (statusCode !== 0) { + throw new Error( + `Docker container returned a non-zero exit code (${statusCode})`, + ); + } + + return { error, statusCode }; +}; diff --git a/packages/backend-common/src/util/index.ts b/packages/backend-common/src/util/index.ts new file mode 100644 index 0000000000..04715824b4 --- /dev/null +++ b/packages/backend-common/src/util/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { runDockerContainer } from './docker'; diff --git a/packages/techdocs-common/src/stages/generate/helpers.test.ts b/packages/techdocs-common/src/stages/generate/helpers.test.ts index fbca6d1ed2..05402d0cf5 100644 --- a/packages/techdocs-common/src/stages/generate/helpers.test.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.test.ts @@ -14,12 +14,10 @@ * limitations under the License. */ import { getVoidLogger } from '@backstage/backend-common'; -import Docker from 'dockerode'; import fs from 'fs-extra'; import mockFs from 'mock-fs'; import os from 'os'; import path, { resolve as resolvePath } from 'path'; -import Stream, { PassThrough } from 'stream'; import { ParsedLocationAnnotation } from '../../helpers'; import { RemoteProtocol } from '../prepare/types'; import { @@ -28,9 +26,7 @@ import { getRepoUrlFromLocationAnnotation, isValidRepoUrlForMkdocs, patchMkdocsYmlPreBuild, - runDockerContainer, storeEtagMetadata, - UserOptions, } from './helpers'; const mockEntity = { @@ -41,8 +37,6 @@ const mockEntity = { }, }; -const mockDocker = new Docker() as jest.Mocked; - const mkdocsYml = fs.readFileSync( resolvePath(__filename, '../__fixtures__/mkdocs.yml'), ); @@ -60,133 +54,6 @@ describe('helpers', () => { }); }); - describe('runDockerContainer', () => { - beforeEach(() => { - jest.spyOn(mockDocker, 'pull').mockImplementation((async ( - _image: string, - _something: any, - handler: (err: Error | undefined, stream: PassThrough) => void, - ) => { - const mockStream = new PassThrough(); - handler(undefined, mockStream); - mockStream.end(); - }) as any); - - jest - .spyOn(mockDocker, 'run') - .mockResolvedValue([{ Error: null, StatusCode: 0 }]); - - jest - .spyOn(mockDocker, 'ping') - .mockResolvedValue(Buffer.from('OK', 'utf-8')); - }); - - const imageName = 'spotify/techdocs'; - const args = ['build', '-d', '/result']; - const docsDir = os.tmpdir(); - const outputDir = os.tmpdir(); - - it('should pull the techdocs docker container', async () => { - await runDockerContainer({ - imageName, - args, - docsDir, - outputDir, - dockerClient: mockDocker, - }); - - expect(mockDocker.pull).toHaveBeenCalledWith( - imageName, - {}, - expect.any(Function), - ); - }); - - it('should run the techdocs docker container', async () => { - await runDockerContainer({ - imageName, - args, - docsDir, - outputDir, - dockerClient: mockDocker, - }); - - expect(mockDocker.run).toHaveBeenCalledWith( - imageName, - args, - expect.any(Stream), - expect.objectContaining({ - Volumes: { - '/content': {}, - '/result': {}, - }, - WorkingDir: '/content', - HostConfig: { - Binds: [`${docsDir}:/content`, `${outputDir}:/result`], - }, - }), - ); - }); - - it('should ping docker to test availability', async () => { - await runDockerContainer({ - imageName, - args, - docsDir, - outputDir, - dockerClient: mockDocker, - }); - - expect(mockDocker.ping).toHaveBeenCalled(); - }); - - it('should pass through the user and group id from the host machine and set the home dir', async () => { - await runDockerContainer({ - imageName, - args, - docsDir, - outputDir, - dockerClient: mockDocker, - }); - - const userOptions: UserOptions = {}; - if (process.getuid && process.getgid) { - userOptions.User = `${process.getuid()}:${process.getgid()}`; - } - - expect(mockDocker.run).toHaveBeenCalledWith( - imageName, - args, - expect.any(Stream), - expect.objectContaining({ - ...userOptions, - }), - ); - }); - - describe('where docker is unavailable', () => { - const dockerError = 'a docker error'; - - beforeEach(() => { - jest.spyOn(mockDocker, 'ping').mockImplementationOnce(() => { - throw new Error(dockerError); - }); - }); - - it('should throw with a descriptive error message including the docker error message', async () => { - await expect( - runDockerContainer({ - imageName, - args, - docsDir, - outputDir, - dockerClient: mockDocker, - }), - ).rejects.toThrow(new RegExp(`.+: ${dockerError}`)); - }); - }); - }); - describe('isValidRepoUrlForMkdocs', () => { it('should return true for valid repo_url values for mkdocs', () => { const validRepoUrls = [ diff --git a/packages/techdocs-common/src/stages/generate/helpers.ts b/packages/techdocs-common/src/stages/generate/helpers.ts index 2035eb8da9..88f42c1e4e 100644 --- a/packages/techdocs-common/src/stages/generate/helpers.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.ts @@ -16,7 +16,6 @@ import { Entity } from '@backstage/catalog-model'; import { spawn } from 'child_process'; -import Docker from 'dockerode'; import fs from 'fs-extra'; import yaml from 'js-yaml'; import { PassThrough, Writable } from 'stream'; @@ -34,16 +33,6 @@ export function getGeneratorKey(entity: Entity): SupportedGeneratorKey { return 'techdocs'; } -type RunDockerContainerOptions = { - imageName: string; - args: string[]; - logStream?: Writable; - docsDir: string; - outputDir: string; - dockerClient: Docker; - createOptions?: Docker.ContainerCreateOptions; -}; - export type RunCommandOptions = { command: string; args: string[]; @@ -51,83 +40,6 @@ export type RunCommandOptions = { logStream?: Writable; }; -export type UserOptions = { - User?: string; -}; - -// To be replaced by a runDockerContainer from backend-common -// shared between Scaffolder and TechDocs and any other plugin. -export async function runDockerContainer({ - imageName, - args, - logStream = new PassThrough(), - docsDir, - outputDir, - dockerClient, - createOptions, -}: RunDockerContainerOptions) { - try { - await dockerClient.ping(); - } catch (e) { - throw new Error( - `This operation requires Docker. Docker does not appear to be available. Docker.ping() failed with: ${e.message}`, - ); - } - - await new Promise((resolve, reject) => { - dockerClient.pull(imageName, {}, (err, stream) => { - if (err) return reject(err); - stream.pipe(logStream, { end: false }); - stream.on('end', () => resolve()); - stream.on('error', (error: Error) => reject(error)); - return undefined; - }); - }); - - const userOptions: UserOptions = {}; - // @ts-ignore - if (process.getuid && process.getgid) { - // Files that are created inside the Docker container will be owned by - // root on the host system on non Mac systems, because of reasons. Mainly the fact that - // volume sharing is done using NFS on Mac and actual mounts in Linux world. - // So we set the user in the container as the same user and group id as the host. - // On Windows we don't have process.getuid nor process.getgid - userOptions.User = `${process.getuid()}:${process.getgid()}`; - } - - const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run( - imageName, - args, - logStream, - { - Volumes: { - '/content': {}, - '/result': {}, - }, - WorkingDir: '/content', - HostConfig: { - Binds: [`${docsDir}:/content`, `${outputDir}:/result`], - }, - ...userOptions, - ...createOptions, - }, - ); - - if (error) { - throw new Error( - `Docker failed to run with the following error message: ${error}`, - ); - } - - if (statusCode !== 0) { - throw new Error( - `Docker container returned a non-zero exit code (${statusCode})`, - ); - } - - return { error, statusCode }; -} - /** * * @param options the options object diff --git a/packages/techdocs-common/src/stages/generate/techdocs.ts b/packages/techdocs-common/src/stages/generate/techdocs.ts index 133755ecd2..0dfa952a17 100644 --- a/packages/techdocs-common/src/stages/generate/techdocs.ts +++ b/packages/techdocs-common/src/stages/generate/techdocs.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { runDockerContainer } from '@backstage/backend-common'; import { Config } from '@backstage/config'; import path from 'path'; import { PassThrough } from 'stream'; @@ -22,7 +23,6 @@ import { addBuildTimestampMetadata, patchMkdocsYmlPreBuild, runCommand, - runDockerContainer, storeEtagMetadata, } from './helpers'; import { GeneratorBase, GeneratorRunOptions } from './types'; @@ -78,6 +78,12 @@ export class TechdocsGenerator implements GeneratorBase { ); } + // Directories to bind on container + const mountDirs = { + [inputDir]: '/input', + [outputDir]: '/output', + }; + try { switch (this.options.runGeneratorIn) { case 'local': @@ -96,10 +102,13 @@ export class TechdocsGenerator implements GeneratorBase { case 'docker': await runDockerContainer({ imageName: 'spotify/techdocs', - args: ['build', '-d', '/result'], + args: ['build', '-d', '/output'], logStream, - docsDir: inputDir, - outputDir, + mountDirs, + workingDir: '/input', + // Set the home directory inside the container as something that applications can + // write to, otherwise they will just fail trying to write to / + envVars: { HOME: '/tmp' }, dockerClient, }); this.logger.info( @@ -115,7 +124,7 @@ export class TechdocsGenerator implements GeneratorBase { this.logger.debug( `Failed to generate docs from ${inputDir} into ${outputDir}`, ); - this.logger.debug(`Build failed with error: ${log}`); + this.logger.error(`Build failed with error: ${log}`); throw new Error( `Failed to generate docs from ${inputDir} into ${outputDir} with error ${error.message}`, ); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts index 77368fc1ab..537aac3088 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts @@ -18,16 +18,17 @@ const runDockerContainer = jest.fn(); const runCommand = jest.fn(); const commandExists = jest.fn(); -jest.mock('./helpers', () => ({ runDockerContainer, runCommand })); +jest.mock('./helpers', () => ({ runCommand })); +jest.mock('@backstage/backend-common', () => ({ runDockerContainer })); jest.mock('command-exists-promise', () => commandExists); jest.mock('fs-extra'); -import { CookieCutter } from './cookiecutter'; +import Docker from 'dockerode'; import fs from 'fs-extra'; +import parseGitUrl from 'git-url-parse'; import path from 'path'; import { PassThrough } from 'stream'; -import Docker from 'dockerode'; -import parseGitUrl from 'git-url-parse'; +import { CookieCutter } from './cookiecutter'; describe('CookieCutter Templater', () => { const mockDocker = {} as Docker; @@ -135,6 +136,11 @@ describe('CookieCutter Templater', () => { }; jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing']); + jest + .spyOn(fs, 'realpath') + .mockImplementation((filePath: string | Buffer) => + Promise.resolve(filePath as string), + ); const templater = new CookieCutter(); await templater.run({ @@ -149,12 +155,16 @@ describe('CookieCutter Templater', () => { 'cookiecutter', '--no-input', '-o', - '/result', - '/template', + '/output', + '/input', '--verbose', ], - templateDir: path.join('tempdir', 'template'), - resultDir: path.join('tempdir', 'intermediate'), + envVars: { HOME: '/tmp' }, + mountDirs: { + [path.join('tempdir', 'template')]: '/input', + [path.join('tempdir', 'intermediate')]: '/output', + }, + workingDir: '/input', logStream: undefined, dockerClient: mockDocker, }); @@ -188,12 +198,16 @@ describe('CookieCutter Templater', () => { 'cookiecutter', '--no-input', '-o', - '/result', - '/template', + '/output', + '/input', '--verbose', ], - templateDir: path.join('tempdir', 'template'), - resultDir: path.join('tempdir', 'intermediate'), + envVars: { HOME: '/tmp' }, + mountDirs: { + [path.join('tempdir', 'template')]: '/input', + [path.join('tempdir', 'intermediate')]: '/output', + }, + workingDir: '/input', logStream: stream, dockerClient: mockDocker, }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts index 9568af2a15..b534610a39 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import fs from 'fs-extra'; +import { runDockerContainer } from '@backstage/backend-common'; import { JsonValue } from '@backstage/config'; -import { runDockerContainer, runCommand } from './helpers'; -import { TemplaterBase, TemplaterRunOptions } from '.'; +import fs from 'fs-extra'; import path from 'path'; +import { TemplaterBase, TemplaterRunOptions } from '.'; +import { runCommand } from './helpers'; const commandExists = require('command-exists-promise'); @@ -57,6 +58,12 @@ export class CookieCutter implements TemplaterBase { await fs.writeJSON(path.join(templateDir, 'cookiecutter.json'), cookieInfo); + // Directories to bind on container + const mountDirs = { + [templateDir]: '/input', + [intermediateDir]: '/output', + }; + const cookieCutterInstalled = await commandExists('cookiecutter'); if (cookieCutterInstalled) { await runCommand({ @@ -71,12 +78,15 @@ export class CookieCutter implements TemplaterBase { 'cookiecutter', '--no-input', '-o', - '/result', - '/template', + '/output', + '/input', '--verbose', ], - templateDir, - resultDir: intermediateDir, + mountDirs, + workingDir: '/input', + // Set the home directory inside the container as something that applications can + // write to, otherwise they will just fail trying to write to / + envVars: { HOME: '/tmp' }, logStream, dockerClient, }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/index.ts index 5e957e0504..96050a14f9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/index.ts @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { runDockerContainer } from '@backstage/backend-common'; import fs from 'fs-extra'; -import { runDockerContainer } from '../helpers'; -import { TemplaterBase, TemplaterRunOptions } from '..'; import path from 'path'; import * as yaml from 'yaml'; +import { TemplaterBase, TemplaterRunOptions } from '..'; // TODO(blam): Replace with the universal import from github-actions after a release // As it will break the E2E without it @@ -40,7 +40,10 @@ export class CreateReactAppTemplater implements TemplaterBase { const intermediateDir = path.join(workspacePath, 'template'); await fs.ensureDir(intermediateDir); - const resultDir = path.join(workspacePath, 'result'); + const mountDirs = { + [intermediateDir]: '/template', + [intermediateDir]: '/result', + }; await runDockerContainer({ imageName: 'node:lts-alpine', @@ -49,10 +52,12 @@ export class CreateReactAppTemplater implements TemplaterBase { componentName as string, withTypescript ? ' --template typescript' : '', ], - templateDir: intermediateDir, - resultDir: intermediateDir, + mountDirs, logStream: logStream, dockerClient: dockerClient, + // Set the home directory inside the container as something that applications can + // write to, otherwise they will just fail trying to write to / + envVars: { HOME: '/tmp' }, createOptions: { Entrypoint: ['npx'], WorkingDir: '/result', @@ -67,6 +72,7 @@ export class CreateReactAppTemplater implements TemplaterBase { throw new Error('No data generated by cookiecutter'); } + const resultDir = path.join(workspacePath, 'result'); await fs.move(path.join(intermediateDir, generated), resultDir); const extraAnnotations: Record = {}; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts deleted file mode 100644 index 57e47f4c27..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import Stream, { PassThrough } from 'stream'; -import os from 'os'; -import fs from 'fs'; -import Docker from 'dockerode'; -import { UserOptions, runDockerContainer } from './helpers'; - -describe('helpers', () => { - const mockDocker = new Docker() as jest.Mocked; - - beforeEach(() => { - jest - .spyOn(mockDocker, 'run') - .mockResolvedValue([{ Error: null, StatusCode: 0 }]); - jest.spyOn(mockDocker, 'pull').mockImplementation((async ( - _image: string, - _something: any, - handler: (err: Error | undefined, stream: PassThrough) => void, - ) => { - const mockStream = new PassThrough(); - handler(undefined, mockStream); - mockStream.end(); - }) as any); - }); - - describe('runDockerContainer', () => { - const imageName = 'blam/github:ben'; - const args = ['bash', '-c', 'echo lol']; - const templateDir = os.tmpdir(); - const resultDir = os.tmpdir(); - - it('will pull the docker container before running', async () => { - await runDockerContainer({ - imageName, - args, - templateDir, - resultDir, - dockerClient: mockDocker, - }); - - expect(mockDocker.pull).toHaveBeenCalledWith( - imageName, - {}, - expect.any(Function), - ); - }); - it('should call the dockerClient run command with the correct arguments passed through', async () => { - await runDockerContainer({ - imageName, - args, - templateDir, - resultDir, - dockerClient: mockDocker, - }); - - expect(mockDocker.run).toHaveBeenCalledWith( - imageName, - args, - expect.any(Stream), - expect.objectContaining({ - HostConfig: { - Binds: expect.arrayContaining([ - `${await fs.promises.realpath(templateDir)}:/template`, - `${await fs.promises.realpath(resultDir)}:/result`, - ]), - }, - Volumes: { - '/template': {}, - '/result': {}, - }, - }), - ); - }); - - it('throws a correct error if the templating fails in docker', async () => { - mockDocker.run.mockResolvedValueOnce([ - { - Error: new Error('Something went wrong with docker'), - StatusCode: 0, - }, - ]); - - await expect( - runDockerContainer({ - imageName, - args, - templateDir, - resultDir, - dockerClient: mockDocker, - }), - ).rejects.toThrow(/Something went wrong with docker/); - }); - - it('throws a correct error when the response code of the container is non-zero', async () => { - mockDocker.run.mockResolvedValueOnce([ - { - Error: null, - StatusCode: 123, - }, - ]); - - await expect( - runDockerContainer({ - imageName, - args, - templateDir, - resultDir, - dockerClient: mockDocker, - }), - ).rejects.toThrow( - /Docker container returned a non-zero exit code \(123\)/, - ); - }); - - it('should pass through the user and group id from the host machine and set the home dir', async () => { - await runDockerContainer({ - imageName, - args, - templateDir, - resultDir, - dockerClient: mockDocker, - }); - - const userOptions: UserOptions = {}; - if (process.getuid && process.getgid) { - userOptions.User = `${process.getuid()}:${process.getgid()}`; - } - - expect(mockDocker.run).toHaveBeenCalledWith( - imageName, - args, - expect.any(Stream), - expect.objectContaining({ - ...userOptions, - Env: ['HOME=/tmp'], - }), - ); - }); - - it('should pass through the log stream to the docker client', async () => { - const logStream = new PassThrough(); - await runDockerContainer({ - imageName, - args, - templateDir, - resultDir, - logStream, - dockerClient: mockDocker, - }); - - expect(mockDocker.run).toHaveBeenCalledWith( - imageName, - args, - logStream, - expect.objectContaining({ - HostConfig: { - Binds: expect.arrayContaining([ - `${await fs.promises.realpath(templateDir)}:/template`, - `${await fs.promises.realpath(resultDir)}:/result`, - ]), - }, - Volumes: { - '/template': {}, - '/result': {}, - }, - }), - ); - }); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts index 645133a4ae..cb1e8532f2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts @@ -13,22 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Writable, PassThrough } from 'stream'; -import Docker from 'dockerode'; -import fs from 'fs'; -import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { InputError } from '@backstage/backend-common'; +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { spawn } from 'child_process'; - -export type RunDockerContainerOptions = { - imageName: string; - args: string[]; - logStream?: Writable; - resultDir: string; - templateDir: string; - dockerClient: Docker; - createOptions?: Docker.ContainerCreateOptions; -}; +import { PassThrough, Writable } from 'stream'; export type RunCommandOptions = { command: string; @@ -36,10 +24,6 @@ export type RunCommandOptions = { logStream?: Writable; }; -export type UserOptions = { - User?: string; -}; - /** * Gets the templater key to use for templating from the entity * @param entity Template entity @@ -89,80 +73,3 @@ export const runCommand = async ({ }); }); }; - -/** - * - * @param options the options object - * @param options.imageName the image to run - * @param options.args the arguments to pass the container - * @param options.logStream the log streamer to capture log messages - * @param options.resultDir the /result path inside the container - * @param options.templateDir the /template path inside the container - * @param options.dockerClient the dockerClient to use - */ -export const runDockerContainer = async ({ - imageName, - args, - logStream = new PassThrough(), - resultDir, - templateDir, - dockerClient, - createOptions = {}, -}: RunDockerContainerOptions) => { - await new Promise((resolve, reject) => { - dockerClient.pull(imageName, {}, (err, stream) => { - if (err) return reject(err); - stream.pipe(logStream, { end: false }); - stream.on('end', () => resolve()); - stream.on('error', (error: Error) => reject(error)); - return undefined; - }); - }); - - const userOptions: UserOptions = {}; - // @ts-ignore - if (process.getuid && process.getgid) { - // Files that are created inside the Docker container will be owned by - // root on the host system on non Mac systems, because of reasons. Mainly the fact that - // volume sharing is done using NFS on Mac and actual mounts in Linux world. - // So we set the user in the container as the same user and group id as the host. - // On Windows we don't have process.getuid nor process.getgid - userOptions.User = `${process.getuid()}:${process.getgid()}`; - } - - const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run( - imageName, - args, - logStream, - { - Volumes: { '/result': {}, '/template': {} }, - HostConfig: { - Binds: [ - // Need to use realpath here as Docker mounting does not like - // symlinks for binding volumes - `${await fs.promises.realpath(resultDir)}:/result`, - `${await fs.promises.realpath(templateDir)}:/template`, - ], - }, - ...userOptions, - // Set the home directory inside the container as something that applications can - // write to, otherwise they will just flop and fail trying to write to / - Env: ['HOME=/tmp'], - ...createOptions, - }, - ); - - if (error) { - throw new Error( - `Docker failed to run with the following error message: ${error}`, - ); - } - - if (statusCode !== 0) { - throw new Error( - `Docker container returned a non-zero exit code (${statusCode})`, - ); - } - - return { error, statusCode }; -};