diff --git a/packages/backend/package.json b/packages/backend/package.json index ec54da358c..3be1f5ee3e 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -26,6 +26,7 @@ "@backstage/plugin-identity-backend": "^0.1.1-alpha.12", "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.12", "@backstage/plugin-sentry-backend": "^0.1.1-alpha.12", + "dockerode": "^3.2.0", "express": "^4.17.1", "knex": "^0.21.1", "sqlite3": "^4.2.0", @@ -35,6 +36,7 @@ "@backstage/cli": "^0.1.1-alpha.12", "@types/express": "^4.17.6", "@types/express-serve-static-core": "^4.17.5", - "@types/helmet": "^0.0.47" + "@types/helmet": "^0.0.47", + "@types/dockerode": "^2.5.32" } } diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index 95d28eacb1..1b0ee46969 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -22,15 +22,17 @@ import { Preparers, } from '@backstage/plugin-scaffolder-backend'; import type { PluginEnvironment } from '../types'; +import Docker from 'dockerode'; export default async function createPlugin({ logger }: PluginEnvironment) { const templater = new CookieCutter(); const filePreparer = new FilePreparer(); const githubPreparer = new GithubPreparer(); const preparers = new Preparers(); + const dockerClient = new Docker(); preparers.register('file', filePreparer); preparers.register('github', githubPreparer); - return await createRouter({ preparers, templater, logger }); + return await createRouter({ preparers, templater, logger, dockerClient }); } diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 459f70463c..c8c40d7b67 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -23,7 +23,9 @@ "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.12", "@backstage/catalog-model": "^0.1.1-alpha.12", + "@backstage/config": "^0.1.1-alpha.12", "@types/express": "^4.17.6", + "@types/dockerode": "^2.5.32", "compression": "^1.7.4", "cors": "^2.8.5", "dockerode": "^3.2.0", diff --git a/plugins/scaffolder-backend/sample-templates/react-ssr-template/cookiecutter.json b/plugins/scaffolder-backend/sample-templates/react-ssr-template/cookiecutter.json new file mode 100644 index 0000000000..bc6e4b5347 --- /dev/null +++ b/plugins/scaffolder-backend/sample-templates/react-ssr-template/cookiecutter.json @@ -0,0 +1,3 @@ +{ + "_copy_without_render": [".github/workflows/*"] +} diff --git a/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/post_gen_project.sh b/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/post_gen_project.sh index 033a102fed..75c35f2034 100644 --- a/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/post_gen_project.sh +++ b/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/post_gen_project.sh @@ -1,12 +1,7 @@ -#!/bin/bash +#!/bin/sh -# package name is "__component_id__" so that yarn doesn't throw an error -# about invalid characters when running yarn commands. here we replace it with the actual name -sed -i -e "s/__component_id__/{{ cookiecutter.component_id }}/g" package.json - -# node_modules was moved out of the template folder, during the pre_gen hook, -# to avoid cookie_cutter from copying all of them. time to move it back -mv ../../node_modules.tmp ../../\{\{cookiecutter.component_id\}\}/node_modules 2>/dev/null ||: - -# move back the build directory that was moved out in the pre_gen hook (if it exists) -mv ../../build.tmp ../../\{\{cookiecutter.component_id\}\}/build 2>/dev/null ||: +# Move all template files to the root folder +mv ./* ../ +cd .. +rm -rf {{cookiecutter.component_id}} +# # # # # # # # diff --git a/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/pre_gen_project.sh b/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/pre_gen_project.sh deleted file mode 100644 index 142fda9ad9..0000000000 --- a/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/pre_gen_project.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# no way to ignore files in cookiecutter, so move node_modules out while building -# to avoid cookiecutter from copying all of them -mv ../../\{\{cookiecutter.component_id\}\}/node_modules ../../node_modules.tmp 2>/dev/null ||: - -# cookicutter really doesn't like the next.js build directory, so if the app has -# been built from inside the template folder, that folders needs to be moved out as well -mv ../../\{\{cookiecutter.component_id\}\}/build ../../build.tmp 2>/dev/null ||: diff --git a/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml b/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml index 17f9ebbf7d..6834f0d1b6 100644 --- a/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml @@ -7,4 +7,3 @@ metadata: spec: type: cookiecutter path: '.' - diff --git a/plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/package.json b/plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/package.json index a0ae26e267..6ae3e63a17 100644 --- a/plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/package.json +++ b/plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/package.json @@ -1,5 +1,5 @@ { - "name": "__component_id__", + "name": "{{ cookiecutter.component_id }}", "version": "0.0.0", "description": "{{ cookiecutter.description }}", "license": "UNLICENSED", diff --git a/plugins/scaffolder-backend/scripts/Dockerfile b/plugins/scaffolder-backend/scripts/Dockerfile new file mode 100644 index 0000000000..b33c7cbe64 --- /dev/null +++ b/plugins/scaffolder-backend/scripts/Dockerfile @@ -0,0 +1,11 @@ +FROM alpine:3.7 + +RUN apk add --update \ + git \ + python \ + python-dev \ + py-pip \ + g++ && \ + pip install cookiecutter && \ + apk del g++ py-pip python-dev && \ + rm -rf /var/cache/apk/* diff --git a/plugins/scaffolder-backend/src/index.ts b/plugins/scaffolder-backend/src/index.ts index c461bfede6..0a0a4cb95f 100644 --- a/plugins/scaffolder-backend/src/index.ts +++ b/plugins/scaffolder-backend/src/index.ts @@ -16,3 +16,4 @@ export * from './scaffolder'; export * from './service/router'; + diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts new file mode 100644 index 0000000000..7c78badff1 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('./helpers', () => ({ runDockerContainer: jest.fn() })); + +import { CookieCutter } from './cookiecutter'; +import fs from 'fs-extra'; +import os from 'os'; +import { RunDockerContainerOptions } from './helpers'; +import { PassThrough } from 'stream'; +import Docker from 'dockerode'; + +describe('CookieCutter Templater', () => { + const cookie = new CookieCutter(); + const mockDocker = {} as Docker; + const { + runDockerContainer, + }: { + runDockerContainer: jest.Mock; + } = require('./helpers'); + + beforeEach(async () => { + jest.clearAllMocks(); + + await fs.remove(`${os.tmpdir()}/cookiecutter.json`); + }); + + it('should write a cookiecutter.json file with the values from the entitiy', async () => { + const tempdir = os.tmpdir(); + + const values = { + component_id: 'test', + description: 'description', + }; + + await cookie.run({ directory: tempdir, values, dockerClient: mockDocker }); + + const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`); + + expect(cookieCutterJson).toEqual(expect.objectContaining(values)); + }); + + it('should merge any value that is in the cookiecutter.json path already', async () => { + const tempdir = os.tmpdir(); + const existingJson = { + _copy_without_render: ['./github/workflows/*'], + }; + await fs.writeJSON(`${tempdir}/cookiecutter.json`, existingJson); + + const values = { + component_id: 'hello', + description: 'im something cool', + }; + + await cookie.run({ directory: tempdir, values, dockerClient: mockDocker }); + + const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`); + + expect(cookieCutterJson).toEqual({ ...existingJson, ...values }); + }); + + it('should throw an error if the cookiecutter json is malformed and not missing', async () => { + const tempdir = os.tmpdir(); + + await fs.writeFile(`${tempdir}/cookiecutter.json`, "{'"); + + const values = { + component_id: 'hello', + description: 'im something cool', + }; + + await expect( + cookie.run({ directory: tempdir, values, dockerClient: mockDocker }), + ).rejects.toThrow(/Unexpected token ' in JSON at position 1/); + }); + + it('should run the correct docker container with the correct bindings for the volumes', async () => { + const tempdir = os.tmpdir(); + + const values = { + component_id: 'test', + description: 'description', + }; + + await cookie.run({ directory: tempdir, values, dockerClient: mockDocker }); + + expect(runDockerContainer).toHaveBeenCalledWith({ + imageName: 'backstage/cookiecutter', + args: ['cookiecutter', '--no-input', '-o', '/result', '/template'], + templateDir: tempdir, + resultDir: `${tempdir}/result`, + logStream: undefined, + dockerClient: mockDocker, + }); + }); + it('should return the result path to the end templated folder', async () => { + const tempdir = os.tmpdir(); + + const values = { + component_id: 'test', + description: 'description', + }; + + const path = await cookie.run({ + directory: tempdir, + values, + dockerClient: mockDocker, + }); + + expect(path).toBe(`${tempdir}/result`); + }); + + it('should pass through the streamer to the run docker helper', async () => { + const stream = new PassThrough(); + + const tempdir = os.tmpdir(); + + const values = { + component_id: 'test', + description: 'description', + }; + + await cookie.run({ + directory: tempdir, + values, + logStream: stream, + dockerClient: mockDocker, + }); + + expect(runDockerContainer).toHaveBeenCalledWith({ + imageName: 'backstage/cookiecutter', + args: ['cookiecutter', '--no-input', '-o', '/result', '/template'], + templateDir: tempdir, + resultDir: `${tempdir}/result`, + logStream: stream, + dockerClient: mockDocker, + }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts index ee5265fde8..a95eee7fb4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts @@ -16,17 +16,52 @@ import { TemplaterBase, TemplaterRunOptions } from '.'; * limitations under the License. */ import fs from 'fs-extra'; +import { JsonValue } from '@backstage/config'; +import { runDockerContainer } from './helpers'; export class CookieCutter implements TemplaterBase { + private async fetchTemplateCookieCutter( + directory: string, + ): Promise> { + try { + return await fs.readJSON(`${directory}/cookiecutter.json`); + } catch (ex) { + if (ex.code !== 'ENOENT') { + throw ex; + } + + return {}; + } + } + public async run(options: TemplaterRunOptions): Promise { - // first we need to make cookiecutter.json in the directory provided with the input values. + // First lets grab the default cookiecutter.json file + const cookieCutterJson = await this.fetchTemplateCookieCutter( + options.directory, + ); + const cookieInfo = { - _copy_without_render: ['.github/workflows/*'], + ...cookieCutterJson, ...options.values, }; await fs.writeJSON(`${options.directory}/cookiecutter.json`, cookieInfo); - return ''; - // run cookie cutter with new json + + const templateDir = options.directory; + + // TODO(blam): This should be an entirely different directory on the host machine + // not in the template directory + const resultDir = `${templateDir}/result`; + + await runDockerContainer({ + imageName: 'backstage/cookiecutter', + args: ['cookiecutter', '--no-input', '-o', '/result', '/template'], + templateDir, + resultDir, + logStream: options.logStream, + dockerClient: options.dockerClient, + }); + + return resultDir; } } diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts new file mode 100644 index 0000000000..d27eb4f93f --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts @@ -0,0 +1,135 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Stream, { PassThrough } from 'stream'; +import os from 'os'; +import fs from 'fs'; +import Docker from 'dockerode'; +import { runDockerContainer } from './helpers'; + +describe('helpers', () => { + const mockDocker = new Docker() as jest.Mocked; + + beforeEach(() => { + jest + .spyOn(mockDocker, 'run') + .mockResolvedValue([{ Error: null, StatusCode: 0 }]); + }); + + describe('runDockerContainer', () => { + const imageName = 'blam/github:ben'; + const args = ['bash', '-c', 'echo lol']; + const templateDir = os.tmpdir(); + const resultDir = os.tmpdir(); + + it('should call the dockerClient run command with the correct arguments passed through', async () => { + await runDockerContainer({ + imageName, + args, + templateDir, + resultDir, + dockerClient: mockDocker, + }); + + expect(mockDocker.run).toHaveBeenCalledWith( + imageName, + args, + expect.any(Stream), + expect.objectContaining({ + HostConfig: { + Binds: expect.arrayContaining([ + `${await fs.promises.realpath(templateDir)}:/template`, + `${await fs.promises.realpath(resultDir)}:/result`, + ]), + }, + Volumes: { + '/template': {}, + '/result': {}, + }, + }), + ); + }); + + it('throws a correct error if the templating fails in docker', async () => { + mockDocker.run.mockResolvedValueOnce([ + { + Error: new Error('Something went wrong with docker'), + StatusCode: 0, + }, + ]); + + await expect( + runDockerContainer({ + imageName, + args, + templateDir, + resultDir, + dockerClient: mockDocker, + }), + ).rejects.toThrow(/Something went wrong with docker/); + }); + + it('throws a correct error when the response code of the container is non-zero', async () => { + mockDocker.run.mockResolvedValueOnce([ + { + Error: null, + StatusCode: 123, + }, + ]); + + await expect( + runDockerContainer({ + imageName, + args, + templateDir, + resultDir, + dockerClient: mockDocker, + }), + ).rejects.toThrow( + /Docker container returned a non-zero exit code \(123\)/, + ); + }); + + it('should pass through the log stream to the docker client', async () => { + const logStream = new PassThrough(); + await runDockerContainer({ + imageName, + args, + templateDir, + resultDir, + logStream, + dockerClient: mockDocker, + }); + + expect(mockDocker.run).toHaveBeenCalledWith( + imageName, + args, + logStream, + expect.objectContaining({ + HostConfig: { + Binds: expect.arrayContaining([ + `${await fs.promises.realpath(templateDir)}:/template`, + `${await fs.promises.realpath(resultDir)}:/result`, + ]), + }, + Volumes: { + '/template': {}, + '/result': {}, + }, + }), + ); + }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts new file mode 100644 index 0000000000..2856a016e4 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts @@ -0,0 +1,77 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Writable, PassThrough } from 'stream'; +import Docker from 'dockerode'; +import fs from 'fs'; + +export type RunDockerContainerOptions = { + imageName: string; + args: string[]; + logStream?: Writable; + resultDir: string; + templateDir: string; + dockerClient: Docker; +}; + +/** + * + * @param options the options object + * @param options.imageName the image to run + * @param options.args the arguments to pass the container + * @param options.logStream the log streamer to capture log messages + * @param options.resultDir the /result path inside the container + * @param options.templateDir the /template path inside the container + * @param options.dockerClient the dockerClient to use + */ +export const runDockerContainer = async ({ + imageName, + args, + logStream = new PassThrough(), + resultDir, + templateDir, + dockerClient, +}: RunDockerContainerOptions) => { + const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run( + imageName, + args, + logStream, + { + Volumes: { '/result': {}, '/template': {} }, + HostConfig: { + Binds: [ + // Need to use realpath here as Docker mounting does not like + // symlinks for binding volumes + `${await fs.promises.realpath(resultDir)}:/result`, + `${await fs.promises.realpath(templateDir)}:/template`, + ], + }, + }, + ); + + if (error) { + throw new Error( + `Docker failed to run with the following error message: ${error}`, + ); + } + + if (statusCode !== 0) { + throw new Error( + `Docker container returned a non-zero exit code (${statusCode})`, + ); + } + + return { error, statusCode }; +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/index.ts b/plugins/scaffolder-backend/src/scaffolder/templater/index.ts index 80570a1c48..8885e65308 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/templater/index.ts @@ -14,6 +14,9 @@ * limitations under the License. */ +import type { Writable } from 'stream'; +import Docker from 'dockerode'; + export interface RequiredTemplateValues { component_id: string; } @@ -21,12 +24,14 @@ export interface RequiredTemplateValues { export interface TemplaterRunOptions { directory: string; values: RequiredTemplateValues & object; + logStream?: Writable; + dockerClient: Docker; } -export abstract class TemplaterBase { +export type TemplaterBase = { // runs the templating with the values and returns the directory to push the VCS - abstract async run(opts: TemplaterRunOptions): Promise; -} + run(opts: TemplaterRunOptions): Promise; +}; export interface TemplaterConfig { templater?: TemplaterBase; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 17b374f1e6..ab39864e76 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -19,18 +19,20 @@ import Router from 'express-promise-router'; import express from 'express'; import { PreparerBuilder, TemplaterBase } from '../scaffolder'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import Docker from 'dockerode'; export interface RouterOptions { preparers: PreparerBuilder; templater: TemplaterBase; logger: Logger; + dockerClient: Docker; } export async function createRouter( options: RouterOptions, ): Promise { const router = Router(); - const { preparers, templater, logger: parentLogger } = options; + const { preparers, templater, logger: parentLogger, dockerClient } = options; const logger = parentLogger.child({ plugin: 'scaffolder' }); router.post('/v1/jobs', async (_, res) => { @@ -66,12 +68,16 @@ export async function createRouter( const preparer = preparers.get(mockEntity); // Run the preparer for the mock entity to produce a temporary directory with template in - const path = await preparer.prepare(mockEntity); + const skeletonPath = await preparer.prepare(mockEntity); // Run the templater on the mock directory with values from the post body - await templater.run({ directory: path, values: { component_id: 'test' } }); + const templatedPath = await templater.run({ + directory: skeletonPath, + values: { component_id: 'test' }, + dockerClient, + }); - console.warn(path); + console.warn(templatedPath); }); const app = express(); diff --git a/yarn.lock b/yarn.lock index dc34848cba..026946b1e5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3433,6 +3433,13 @@ resolved "https://registry.npmjs.org/@types/diff/-/diff-4.0.2.tgz#2e9bb89f9acc3ab0108f0f3dc4dbdcf2fff8a99c" integrity sha512-mIenTfsIe586/yzsyfql69KRnA75S8SVXQbTLpDejRrjH0QSJcpu3AUOi/Vjnt9IOsXKxPhJfGpQUNMueIU1fQ== +"@types/dockerode@^2.5.32": + version "2.5.32" + resolved "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.32.tgz#52d3628f605f8ea65202541c59a8a6dd166384fd" + integrity sha512-TfaGOoOHxsjkWRj2sPoQ3FLmTC5mVMhZ4kzZy13U7mjtIDoloE4e7AMj5jPLbffWB6Csy5DF5e0lC9M+tnKz/A== + dependencies: + "@types/node" "*" + "@types/eslint-visitor-keys@^1.0.0": version "1.0.0" resolved "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz#1ee30d79544ca84d68d4b3cdb0af4f205663dd2d"