Merge pull request #1418 from spotify/mob/cookiecutter-templater

Implement the CookieCutter Templater with Docker

packages/backend/package.json

@@ -26,6 +26,7 @@
     "@backstage/plugin-identity-backend": "^0.1.1-alpha.12",
     "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.12",
     "@backstage/plugin-sentry-backend": "^0.1.1-alpha.12",
+    "dockerode": "^3.2.0",
     "express": "^4.17.1",
     "knex": "^0.21.1",
     "sqlite3": "^4.2.0",
@@ -35,6 +36,7 @@
     "@backstage/cli": "^0.1.1-alpha.12",
     "@types/express": "^4.17.6",
     "@types/express-serve-static-core": "^4.17.5",
-    "@types/helmet": "^0.0.47"
+    "@types/helmet": "^0.0.47",
+    "@types/dockerode": "^2.5.32"
   }
 }

packages/backend/src/plugins/scaffolder.ts

@@ -22,15 +22,17 @@ import {
   Preparers,
 } from '@backstage/plugin-scaffolder-backend';
 import type { PluginEnvironment } from '../types';
+import Docker from 'dockerode';
 
 export default async function createPlugin({ logger }: PluginEnvironment) {
   const templater = new CookieCutter();
   const filePreparer = new FilePreparer();
   const githubPreparer = new GithubPreparer();
   const preparers = new Preparers();
+  const dockerClient = new Docker();
 
   preparers.register('file', filePreparer);
   preparers.register('github', githubPreparer);
 
-  return await createRouter({ preparers, templater, logger });
+  return await createRouter({ preparers, templater, logger, dockerClient });
 }

plugins/scaffolder-backend/package.json

@@ -23,7 +23,9 @@
   "dependencies": {
     "@backstage/backend-common": "^0.1.1-alpha.12",
     "@backstage/catalog-model": "^0.1.1-alpha.12",
+    "@backstage/config": "^0.1.1-alpha.12",
     "@types/express": "^4.17.6",
+    "@types/dockerode": "^2.5.32",
     "compression": "^1.7.4",
     "cors": "^2.8.5",
     "dockerode": "^3.2.0",

plugins/scaffolder-backend/sample-templates/react-ssr-template/cookiecutter.json

@@ -0,0 +1,3 @@
+{
+  "_copy_without_render": [".github/workflows/*"]
+}

plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/post_gen_project.sh

@@ -1,12 +1,7 @@
-#!/bin/bash
+#!/bin/sh
 
-# package name is "__component_id__" so that yarn doesn't throw an error
-# about invalid characters when running yarn commands. here we replace it with the actual name
-sed -i -e "s/__component_id__/{{ cookiecutter.component_id }}/g" package.json
-
-# node_modules was moved out of the template folder, during the pre_gen hook,
-# to avoid cookie_cutter from copying all of them. time to move it back
-mv ../../node_modules.tmp ../../\{\{cookiecutter.component_id\}\}/node_modules 2>/dev/null ||:
-
-# move back the build directory that was moved out in the pre_gen hook (if it exists)
-mv ../../build.tmp ../../\{\{cookiecutter.component_id\}\}/build 2>/dev/null ||:
+# Move all template files to the root folder
+mv ./* ../
+cd ..
+rm -rf {{cookiecutter.component_id}}
+# # # # # # # # 

plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/pre_gen_project.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-# no way to ignore files in cookiecutter, so move node_modules out while building
-# to avoid cookiecutter from copying all of them
-mv ../../\{\{cookiecutter.component_id\}\}/node_modules ../../node_modules.tmp  2>/dev/null ||:
-
-# cookicutter really doesn't like the next.js build directory, so if the app has
-# been built from inside the template folder, that folders needs to be moved out as well
-mv ../../\{\{cookiecutter.component_id\}\}/build ../../build.tmp  2>/dev/null ||:

plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml

@@ -7,4 +7,3 @@ metadata:
 spec:
   type: cookiecutter
   path: '.'
-

plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "__component_id__",
+  "name": "{{ cookiecutter.component_id }}",
   "version": "0.0.0",
   "description": "{{ cookiecutter.description }}",
   "license": "UNLICENSED",

plugins/scaffolder-backend/scripts/Dockerfile

@@ -0,0 +1,11 @@
+FROM alpine:3.7
+
+RUN apk add --update \
+	git \
+	python \
+	python-dev \
+	py-pip \
+	g++ && \
+	pip install cookiecutter && \
+	apk del g++ py-pip python-dev && \
+	rm -rf /var/cache/apk/* 

plugins/scaffolder-backend/src/index.ts

@@ -16,3 +16,4 @@
 
 export * from './scaffolder';
 export * from './service/router';
+

plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts

@@ -0,0 +1,151 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+jest.mock('./helpers', () => ({ runDockerContainer: jest.fn() }));
+
+import { CookieCutter } from './cookiecutter';
+import fs from 'fs-extra';
+import os from 'os';
+import { RunDockerContainerOptions } from './helpers';
+import { PassThrough } from 'stream';
+import Docker from 'dockerode';
+
+describe('CookieCutter Templater', () => {
+  const cookie = new CookieCutter();
+  const mockDocker = {} as Docker;
+  const {
+    runDockerContainer,
+  }: {
+    runDockerContainer: jest.Mock<RunDockerContainerOptions>;
+  } = require('./helpers');
+
+  beforeEach(async () => {
+    jest.clearAllMocks();
+
+    await fs.remove(`${os.tmpdir()}/cookiecutter.json`);
+  });
+
+  it('should write a cookiecutter.json file with the values from the entitiy', async () => {
+    const tempdir = os.tmpdir();
+
+    const values = {
+      component_id: 'test',
+      description: 'description',
+    };
+
+    await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
+
+    const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`);
+
+    expect(cookieCutterJson).toEqual(expect.objectContaining(values));
+  });
+
+  it('should merge any value that is in the cookiecutter.json path already', async () => {
+    const tempdir = os.tmpdir();
+    const existingJson = {
+      _copy_without_render: ['./github/workflows/*'],
+    };
+    await fs.writeJSON(`${tempdir}/cookiecutter.json`, existingJson);
+
+    const values = {
+      component_id: 'hello',
+      description: 'im something cool',
+    };
+
+    await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
+
+    const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`);
+
+    expect(cookieCutterJson).toEqual({ ...existingJson, ...values });
+  });
+
+  it('should throw an error if the cookiecutter json is malformed and not missing', async () => {
+    const tempdir = os.tmpdir();
+
+    await fs.writeFile(`${tempdir}/cookiecutter.json`, "{'");
+
+    const values = {
+      component_id: 'hello',
+      description: 'im something cool',
+    };
+
+    await expect(
+      cookie.run({ directory: tempdir, values, dockerClient: mockDocker }),
+    ).rejects.toThrow(/Unexpected token ' in JSON at position 1/);
+  });
+
+  it('should run the correct docker container with the correct bindings for the volumes', async () => {
+    const tempdir = os.tmpdir();
+
+    const values = {
+      component_id: 'test',
+      description: 'description',
+    };
+
+    await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
+
+    expect(runDockerContainer).toHaveBeenCalledWith({
+      imageName: 'backstage/cookiecutter',
+      args: ['cookiecutter', '--no-input', '-o', '/result', '/template'],
+      templateDir: tempdir,
+      resultDir: `${tempdir}/result`,
+      logStream: undefined,
+      dockerClient: mockDocker,
+    });
+  });
+  it('should return the result path to the end templated folder', async () => {
+    const tempdir = os.tmpdir();
+
+    const values = {
+      component_id: 'test',
+      description: 'description',
+    };
+
+    const path = await cookie.run({
+      directory: tempdir,
+      values,
+      dockerClient: mockDocker,
+    });
+
+    expect(path).toBe(`${tempdir}/result`);
+  });
+
+  it('should pass through the streamer to the run docker helper', async () => {
+    const stream = new PassThrough();
+
+    const tempdir = os.tmpdir();
+
+    const values = {
+      component_id: 'test',
+      description: 'description',
+    };
+
+    await cookie.run({
+      directory: tempdir,
+      values,
+      logStream: stream,
+      dockerClient: mockDocker,
+    });
+
+    expect(runDockerContainer).toHaveBeenCalledWith({
+      imageName: 'backstage/cookiecutter',
+      args: ['cookiecutter', '--no-input', '-o', '/result', '/template'],
+      templateDir: tempdir,
+      resultDir: `${tempdir}/result`,
+      logStream: stream,
+      dockerClient: mockDocker,
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts

@@ -16,17 +16,52 @@ import { TemplaterBase, TemplaterRunOptions } from '.';
  * limitations under the License.
  */
 import fs from 'fs-extra';
+import { JsonValue } from '@backstage/config';
+import { runDockerContainer } from './helpers';
 
 export class CookieCutter implements TemplaterBase {
+  private async fetchTemplateCookieCutter(
+    directory: string,
+  ): Promise<Record<string, JsonValue>> {
+    try {
+      return await fs.readJSON(`${directory}/cookiecutter.json`);
+    } catch (ex) {
+      if (ex.code !== 'ENOENT') {
+        throw ex;
+      }
+
+      return {};
+    }
+  }
+
   public async run(options: TemplaterRunOptions): Promise<string> {
-    // first we need to make cookiecutter.json in the directory provided with the input values.
+    // First lets grab the default cookiecutter.json file
+    const cookieCutterJson = await this.fetchTemplateCookieCutter(
+      options.directory,
+    );
+
     const cookieInfo = {
-      _copy_without_render: ['.github/workflows/*'],
+      ...cookieCutterJson,
       ...options.values,
     };
 
     await fs.writeJSON(`${options.directory}/cookiecutter.json`, cookieInfo);
-    return '';
-    // run cookie cutter with new json
+
+    const templateDir = options.directory;
+
+    // TODO(blam): This should be an entirely different directory on the host machine
+    // not in the template directory
+    const resultDir = `${templateDir}/result`;
+
+    await runDockerContainer({
+      imageName: 'backstage/cookiecutter',
+      args: ['cookiecutter', '--no-input', '-o', '/result', '/template'],
+      templateDir,
+      resultDir,
+      logStream: options.logStream,
+      dockerClient: options.dockerClient,
+    });
+
+    return resultDir;
   }
 }

plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts

@@ -0,0 +1,135 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import Stream, { PassThrough } from 'stream';
+import os from 'os';
+import fs from 'fs';
+import Docker from 'dockerode';
+import { runDockerContainer } from './helpers';
+
+describe('helpers', () => {
+  const mockDocker = new Docker() as jest.Mocked<Docker>;
+
+  beforeEach(() => {
+    jest
+      .spyOn(mockDocker, 'run')
+      .mockResolvedValue([{ Error: null, StatusCode: 0 }]);
+  });
+
+  describe('runDockerContainer', () => {
+    const imageName = 'blam/github:ben';
+    const args = ['bash', '-c', 'echo lol'];
+    const templateDir = os.tmpdir();
+    const resultDir = os.tmpdir();
+
+    it('should call the dockerClient run command with the correct arguments passed through', async () => {
+      await runDockerContainer({
+        imageName,
+        args,
+        templateDir,
+        resultDir,
+        dockerClient: mockDocker,
+      });
+
+      expect(mockDocker.run).toHaveBeenCalledWith(
+        imageName,
+        args,
+        expect.any(Stream),
+        expect.objectContaining({
+          HostConfig: {
+            Binds: expect.arrayContaining([
+              `${await fs.promises.realpath(templateDir)}:/template`,
+              `${await fs.promises.realpath(resultDir)}:/result`,
+            ]),
+          },
+          Volumes: {
+            '/template': {},
+            '/result': {},
+          },
+        }),
+      );
+    });
+
+    it('throws a correct error if the templating fails in docker', async () => {
+      mockDocker.run.mockResolvedValueOnce([
+        {
+          Error: new Error('Something went wrong with docker'),
+          StatusCode: 0,
+        },
+      ]);
+
+      await expect(
+        runDockerContainer({
+          imageName,
+          args,
+          templateDir,
+          resultDir,
+          dockerClient: mockDocker,
+        }),
+      ).rejects.toThrow(/Something went wrong with docker/);
+    });
+
+    it('throws a correct error when the response code of the container is non-zero', async () => {
+      mockDocker.run.mockResolvedValueOnce([
+        {
+          Error: null,
+          StatusCode: 123,
+        },
+      ]);
+
+      await expect(
+        runDockerContainer({
+          imageName,
+          args,
+          templateDir,
+          resultDir,
+          dockerClient: mockDocker,
+        }),
+      ).rejects.toThrow(
+        /Docker container returned a non-zero exit code \(123\)/,
+      );
+    });
+
+    it('should pass through the log stream to the docker client', async () => {
+      const logStream = new PassThrough();
+      await runDockerContainer({
+        imageName,
+        args,
+        templateDir,
+        resultDir,
+        logStream,
+        dockerClient: mockDocker,
+      });
+
+      expect(mockDocker.run).toHaveBeenCalledWith(
+        imageName,
+        args,
+        logStream,
+        expect.objectContaining({
+          HostConfig: {
+            Binds: expect.arrayContaining([
+              `${await fs.promises.realpath(templateDir)}:/template`,
+              `${await fs.promises.realpath(resultDir)}:/result`,
+            ]),
+          },
+          Volumes: {
+            '/template': {},
+            '/result': {},
+          },
+        }),
+      );
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Writable, PassThrough } from 'stream';
+import Docker from 'dockerode';
+import fs from 'fs';
+
+export type RunDockerContainerOptions = {
+  imageName: string;
+  args: string[];
+  logStream?: Writable;
+  resultDir: string;
+  templateDir: string;
+  dockerClient: Docker;
+};
+
+/**
+ *
+ * @param options the options object
+ * @param options.imageName the image to run
+ * @param options.args the arguments to pass the container
+ * @param options.logStream the log streamer to capture log messages
+ * @param options.resultDir the /result path inside the container
+ * @param options.templateDir the /template path inside the container
+ * @param options.dockerClient the dockerClient to use
+ */
+export const runDockerContainer = async ({
+  imageName,
+  args,
+  logStream = new PassThrough(),
+  resultDir,
+  templateDir,
+  dockerClient,
+}: RunDockerContainerOptions) => {
+  const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run(
+    imageName,
+    args,
+    logStream,
+    {
+      Volumes: { '/result': {}, '/template': {} },
+      HostConfig: {
+        Binds: [
+          // Need to use realpath here as Docker mounting does not like
+          // symlinks for binding volumes
+          `${await fs.promises.realpath(resultDir)}:/result`,
+          `${await fs.promises.realpath(templateDir)}:/template`,
+        ],
+      },
+    },
+  );
+
+  if (error) {
+    throw new Error(
+      `Docker failed to run with the following error message: ${error}`,
+    );
+  }
+
+  if (statusCode !== 0) {
+    throw new Error(
+      `Docker container returned a non-zero exit code (${statusCode})`,
+    );
+  }
+
+  return { error, statusCode };
+};

plugins/scaffolder-backend/src/scaffolder/templater/index.ts

@@ -14,6 +14,9 @@
  * limitations under the License.
  */
 
+import type { Writable } from 'stream';
+import Docker from 'dockerode';
+
 export interface RequiredTemplateValues {
   component_id: string;
 }
@@ -21,12 +24,14 @@ export interface RequiredTemplateValues {
 export interface TemplaterRunOptions {
   directory: string;
   values: RequiredTemplateValues & object;
+  logStream?: Writable;
+  dockerClient: Docker;
 }
 
-export abstract class TemplaterBase {
+export type TemplaterBase = {
   // runs the templating with the values and returns the directory to push the VCS
-  abstract async run(opts: TemplaterRunOptions): Promise<string>;
-}
+  run(opts: TemplaterRunOptions): Promise<string>;
+};
 
 export interface TemplaterConfig {
   templater?: TemplaterBase;

plugins/scaffolder-backend/src/service/router.ts

@@ -19,18 +19,20 @@ import Router from 'express-promise-router';
 import express from 'express';
 import { PreparerBuilder, TemplaterBase } from '../scaffolder';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import Docker from 'dockerode';
 
 export interface RouterOptions {
   preparers: PreparerBuilder;
   templater: TemplaterBase;
   logger: Logger;
+  dockerClient: Docker;
 }
 
 export async function createRouter(
   options: RouterOptions,
 ): Promise<express.Router> {
   const router = Router();
-  const { preparers, templater, logger: parentLogger } = options;
+  const { preparers, templater, logger: parentLogger, dockerClient } = options;
   const logger = parentLogger.child({ plugin: 'scaffolder' });
 
   router.post('/v1/jobs', async (_, res) => {
@@ -66,12 +68,16 @@ export async function createRouter(
     const preparer = preparers.get(mockEntity);
 
     // Run the preparer for the mock entity to produce a temporary directory with template in
-    const path = await preparer.prepare(mockEntity);
+    const skeletonPath = await preparer.prepare(mockEntity);
 
     // Run the templater on the mock directory with values from the post body
-    await templater.run({ directory: path, values: { component_id: 'test' } });
+    const templatedPath = await templater.run({
+      directory: skeletonPath,
+      values: { component_id: 'test' },
+      dockerClient,
+    });
 
-    console.warn(path);
+    console.warn(templatedPath);
   });
 
   const app = express();

yarn.lock

@@ -3433,6 +3433,13 @@
   resolved "https://registry.npmjs.org/@types/diff/-/diff-4.0.2.tgz#2e9bb89f9acc3ab0108f0f3dc4dbdcf2fff8a99c"
   integrity sha512-mIenTfsIe586/yzsyfql69KRnA75S8SVXQbTLpDejRrjH0QSJcpu3AUOi/Vjnt9IOsXKxPhJfGpQUNMueIU1fQ==
 
+"@types/dockerode@^2.5.32":
+  version "2.5.32"
+  resolved "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.32.tgz#52d3628f605f8ea65202541c59a8a6dd166384fd"
+  integrity sha512-TfaGOoOHxsjkWRj2sPoQ3FLmTC5mVMhZ4kzZy13U7mjtIDoloE4e7AMj5jPLbffWB6Csy5DF5e0lC9M+tnKz/A==
+  dependencies:
+    "@types/node" "*"
+
 "@types/eslint-visitor-keys@^1.0.0":
   version "1.0.0"
   resolved "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz#1ee30d79544ca84d68d4b3cdb0af4f205663dd2d"