refactor(scaffolder): Moving out the dockerish logic into something that is easy to consume for other templaters

plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml

@@ -7,4 +7,3 @@ metadata:
 spec:
   type: cookiecutter
   path: '.'
-

plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts

@@ -13,27 +13,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+jest.mock('./helpers', () => ({ runDockerContainer: jest.fn() }));
+
 import { CookieCutter } from './cookiecutter';
 import fs from 'fs-extra';
 import os from 'os';
-import Stream, { PassThrough } from 'stream';
-
-const mockDocker = {
-  run: jest.fn<any, any>(() => [{ Error: null, StatusCode: 0 }]),
-};
-jest.mock(
-  'dockerode',
-  () =>
-    class {
-      constructor() {
-        return mockDocker;
-      }
-    },
-);
+import { RunDockerContainerOptions } from './helpers';
+import { PassThrough } from 'stream';
 
 describe('CookieCutter Templater', () => {
   const cookie = new CookieCutter();
 
+  const {
+    runDockerContainer,
+  }: {
+    runDockerContainer: jest.Mock<RunDockerContainerOptions>;
+  } = require('./helpers');
+
   beforeEach(() => {
     jest.clearAllMocks();
   });
@@ -82,29 +78,14 @@ describe('CookieCutter Templater', () => {
 
     await cookie.run({ directory: tempdir, values });
 
-    const realpath = await fs.realpath(tempdir);
-
-    // TODO(blam): This might change when we publish our own cookiecutter image
-    // to @backstage/cookiecutter in docker hub.
-    expect(mockDocker.run).toHaveBeenCalledWith(
-      'backstage/cookiecutter',
-      ['cookiecutter', '--no-input', '-o', '/result', '/template'],
-      expect.any(Stream),
-      expect.objectContaining({
-        HostConfig: {
-          Binds: expect.arrayContaining([
-            `${realpath}:/template`,
-            `${realpath}/result:/result`,
-          ]),
-        },
-        Volumes: {
-          '/template': {},
-          '/result': {},
-        },
-      }),
-    );
+    expect(runDockerContainer).toHaveBeenCalledWith({
+      imageName: 'backstage/cookiecutter',
+      args: ['cookiecutter', '--no-input', '-o', '/result', '/template'],
+      templateDir: tempdir,
+      resultDir: `${tempdir}/result`,
+      logStream: undefined,
+    });
   });
-
   it('should return the result path to the end templated folder', async () => {
     const tempdir = os.tmpdir();
 
@@ -115,18 +96,11 @@ describe('CookieCutter Templater', () => {
 
     const path = await cookie.run({ directory: tempdir, values });
 
-    const realpath = await fs.realpath(tempdir);
-
-    expect(path).toBe(`${realpath}/result`);
+    expect(path).toBe(`${tempdir}/result`);
   });
 
-  it('throws a correct error if the templating fails in docker', async () => {
-    mockDocker.run.mockResolvedValueOnce([
-      {
-        Error: new Error('Something went wrong with docker'),
-        StatusCode: 0,
-      },
-    ]);
+  it('should pass through the streamer to the run docker helper', async () => {
+    const stream = new PassThrough();
 
     const tempdir = os.tmpdir();
 
@@ -135,47 +109,14 @@ describe('CookieCutter Templater', () => {
       description: 'description',
     };
 
-    await expect(cookie.run({ directory: tempdir, values })).rejects.toThrow(
-      /Something went wrong with docker/,
-    );
-  });
+    await cookie.run({ directory: tempdir, values, logStream: stream });
 
-  it('uses the passed stream as a log stream', async () => {
-    const logStream = new PassThrough();
-    const tempdir = os.tmpdir();
-
-    const values = {
-      component_id: 'test',
-      description: 'description',
-    };
-
-    await cookie.run({ directory: tempdir, values, logStream });
-
-    expect(mockDocker.run).toHaveBeenCalledWith(
-      expect.any(String),
-      expect.any(Array),
-      logStream,
-      expect.any(Object),
-    );
-  });
-
-  it('throws a correct error if the container returns a non-zero exit code', async () => {
-    mockDocker.run.mockResolvedValueOnce([
-      {
-        Error: null,
-        StatusCode: 1,
-      },
-    ]);
-
-    const tempdir = os.tmpdir();
-
-    const values = {
-      component_id: 'test',
-      description: 'description',
-    };
-
-    await expect(cookie.run({ directory: tempdir, values })).rejects.toThrow(
-      /Docker container returned a non-zero exit code \(1\)/,
-    );
+    expect(runDockerContainer).toHaveBeenCalledWith({
+      imageName: 'backstage/cookiecutter',
+      args: ['cookiecutter', '--no-input', '-o', '/result', '/template'],
+      templateDir: tempdir,
+      resultDir: `${tempdir}/result`,
+      logStream: stream,
+    });
   });
 });

plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts

@@ -16,15 +16,10 @@ import { TemplaterBase, TemplaterRunOptions } from '.';
  * limitations under the License.
  */
 import fs from 'fs-extra';
-import Docker from 'dockerode';
 import { JsonValue } from '@backstage/config';
-import { PassThrough } from 'stream';
-export class CookieCutter implements TemplaterBase {
-  private docker: Docker;
-  constructor() {
-    this.docker = new Docker();
-  }
+import { runDockerContainer } from './helpers';
 
+export class CookieCutter implements TemplaterBase {
   private async fetchTemplateCookieCutter(
     directory: string,
   ): Promise<Record<string, JsonValue>> {
@@ -48,35 +43,20 @@ export class CookieCutter implements TemplaterBase {
 
     await fs.writeJSON(`${options.directory}/cookiecutter.json`, cookieInfo);
 
-    const realTemplatePath = await fs.promises.realpath(options.directory);
-    const outDir = `${realTemplatePath}/result`;
+    const templateDir = options.directory;
 
-    const [
-      { Error: dockerError, StatusCode: containerStatusCode },
-    ] = await this.docker.run(
-      'backstage/cookiecutter',
-      ['cookiecutter', '--no-input', '-o', '/result', '/template'],
-      options.logStream ?? new PassThrough(),
-      {
-        Volumes: { '/result': {}, '/template': {} },
-        HostConfig: {
-          Binds: [`${outDir}:/result`, `${realTemplatePath}:/template`],
-        },
-      },
-    );
+    // TODO(blam): This should be an entirely different directory on the host machine
+    // not in the template directory
+    const resultDir = `${templateDir}/result`;
 
-    if (dockerError) {
-      throw new Error(
-        `Docker failed to run with the following error message: ${dockerError}`,
-      );
-    }
+    await runDockerContainer({
+      imageName: 'backstage/cookiecutter',
+      args: ['cookiecutter', '--no-input', '-o', '/result', '/template'],
+      templateDir,
+      resultDir,
+      logStream: options.logStream,
+    });
 
-    if (containerStatusCode !== 0) {
-      throw new Error(
-        `Docker container returned a non-zero exit code (${containerStatusCode})`,
-      );
-    }
-
-    return outDir;
+    return resultDir;
   }
 }

plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import Stream, { PassThrough } from 'stream';
+import os from 'os';
+import fs from 'fs';
+
+const mockDocker = {
+  run: jest.fn<any, any>(() => [{ Error: null, StatusCode: 0 }]),
+};
+
+jest.mock(
+  'dockerode',
+  () =>
+    class {
+      constructor() {
+        return mockDocker;
+      }
+    },
+);
+
+import { runDockerContainer } from './helpers';
+
+describe('helpers', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('runDockerContainer', () => {
+    const imageName = 'blam/github:ben';
+    const args = ['bash', '-c', 'echo lol'];
+    const templateDir = os.tmpdir();
+    const resultDir = os.tmpdir();
+
+    it('should call the dockerClient run command with the correct arguments passed through', async () => {
+      await runDockerContainer({
+        imageName,
+        args,
+        templateDir,
+        resultDir,
+      });
+
+      expect(mockDocker.run).toHaveBeenCalledWith(
+        imageName,
+        args,
+        expect.any(Stream),
+        expect.objectContaining({
+          HostConfig: {
+            Binds: expect.arrayContaining([
+              `${await fs.promises.realpath(templateDir)}:/template`,
+              `${await fs.promises.realpath(resultDir)}:/result`,
+            ]),
+          },
+          Volumes: {
+            '/template': {},
+            '/result': {},
+          },
+        }),
+      );
+    });
+
+    it('throws a correct error if the templating fails in docker', async () => {
+      mockDocker.run.mockResolvedValueOnce([
+        {
+          Error: new Error('Something went wrong with docker'),
+          StatusCode: 0,
+        },
+      ]);
+
+      await expect(
+        runDockerContainer({ imageName, args, templateDir, resultDir }),
+      ).rejects.toThrow(/Something went wrong with docker/);
+    });
+
+    it('throws a correct error when the response code of the container is non-zero', async () => {
+      mockDocker.run.mockResolvedValueOnce([
+        {
+          Error: null,
+          StatusCode: 123,
+        },
+      ]);
+
+      await expect(
+        runDockerContainer({ imageName, args, templateDir, resultDir }),
+      ).rejects.toThrow(
+        /Docker container returned a non-zero exit code \(123\)/,
+      );
+    });
+
+    it('should pass through the log stream to the docker client', async () => {
+      const logStream = new PassThrough();
+      await runDockerContainer({
+        imageName,
+        args,
+        templateDir,
+        resultDir,
+        logStream,
+      });
+
+      expect(mockDocker.run).toHaveBeenCalledWith(
+        imageName,
+        args,
+        logStream,
+        expect.objectContaining({
+          HostConfig: {
+            Binds: expect.arrayContaining([
+              `${await fs.promises.realpath(templateDir)}:/template`,
+              `${await fs.promises.realpath(resultDir)}:/result`,
+            ]),
+          },
+          Volumes: {
+            '/template': {},
+            '/result': {},
+          },
+        }),
+      );
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import Docker from 'dockerode';
+import { Writable, PassThrough } from 'stream';
+import fs from 'fs';
+
+export type RunDockerContainerOptions = {
+  imageName: string;
+  args: string[];
+  logStream?: Writable;
+  resultDir: string;
+  templateDir: string;
+};
+
+const dockerClient = new Docker();
+/**
+ *
+ * @param options the options object
+ * @param options.imageName the image to run
+ * @param options.args the arguments to pass the container
+ * @param options.logStream the log streamer to capture log messages
+ * @param options.resultDir the /result path inside the container
+ * @param options.templateDir the /template path inside the container
+ */
+export const runDockerContainer = async ({
+  imageName,
+  args,
+  logStream = new PassThrough(),
+  resultDir,
+  templateDir,
+}: RunDockerContainerOptions) => {
+  const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run(
+    imageName,
+    args,
+    logStream,
+    {
+      Volumes: { '/result': {}, '/template': {} },
+      HostConfig: {
+        Binds: [
+          // Need to use realpath here as Docker mounting does not like
+          // symlinks for binding volumes
+          `${await fs.promises.realpath(resultDir)}:/result`,
+          `${await fs.promises.realpath(templateDir)}:/template`,
+        ],
+      },
+    },
+  );
+
+  if (error) {
+    throw new Error(
+      `Docker failed to run with the following error message: ${error}`,
+    );
+  }
+
+  if (statusCode !== 0) {
+    throw new Error(
+      `Docker container returned a non-zero exit code (${statusCode})`,
+    );
+  }
+
+  return { error, statusCode };
+};

plugins/scaffolder-backend/src/scaffolder/templater/templaters.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { PreparerBase, RemoteProtocol, PreparerBuilder } from './types';
+import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import { parseLocationAnnotation } from './helpers';
+
+export class Templaters implements PreparerBuilder {
+  private templaterMap = new Map<RemoteProtocol, PreparerBase>();
+
+  register(protocol: RemoteProtocol, preparer: PreparerBase) {
+    this.templaterMap.set(protocol, preparer);
+  }
+
+  get(template: TemplateEntityV1alpha1): PreparerBase {
+    const { protocol } = parseLocationAnnotation(template);
+    const preparer = this.templaterMap.get(protocol);
+
+    if (!preparer) {
+      throw new Error(`No preparer registered for type: "${protocol}"`);
+    }
+
+    return preparer;
+  }
+}