Merge pull request #1594 from spotify/blam/tidy-up-scaffolder

Use the Templater Property in the Template Definition to choose the templater

packages/backend/package.json

@@ -29,6 +29,7 @@
     "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.13",
     "@backstage/plugin-sentry-backend": "^0.1.1-alpha.13",
     "@backstage/plugin-techdocs-backend": "^0.1.1-alpha.13",
+    "@octokit/rest": "^18.0.0",
     "dockerode": "^3.2.0",
     "express": "^4.17.1",
     "knex": "^0.21.1",

packages/backend/src/plugins/scaffolder.ts

@@ -20,19 +20,34 @@ import {
   FilePreparer,
   GithubPreparer,
   Preparers,
+  GithubPublisher,
+  Templaters,
 } from '@backstage/plugin-scaffolder-backend';
+import { Octokit } from '@octokit/rest';
 import type { PluginEnvironment } from '../types';
 import Docker from 'dockerode';
 
 export default async function createPlugin({ logger }: PluginEnvironment) {
-  const templater = new CookieCutter();
+  const cookiecutterTemplater = new CookieCutter();
+  const templaters = new Templaters();
+  templaters.register('cookiecutter', cookiecutterTemplater);
+
   const filePreparer = new FilePreparer();
   const githubPreparer = new GithubPreparer();
   const preparers = new Preparers();
-  const dockerClient = new Docker();
 
   preparers.register('file', filePreparer);
   preparers.register('github', githubPreparer);
 
-  return await createRouter({ preparers, templater, logger, dockerClient });
+  const githubClient = new Octokit({ auth: process.env.GITHUB_ACCESS_TOKEN });
+  const publisher = new GithubPublisher({ client: githubClient });
+
+  const dockerClient = new Docker();
+  return await createRouter({
+    preparers,
+    templaters,
+    publisher,
+    logger,
+    dockerClient,
+  });
 }

packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts

@@ -32,7 +32,8 @@ describe('TemplateEntityV1alpah1', () => {
         name: 'test',
       },
       spec: {
-        type: 'cookiecutter',
+        type: 'website',
+        templater: 'cookiecutter',
         schema: {
           $schema: 'http://json-schema.org/draft-07/schema#',
           required: ['storePath', 'owner'],
@@ -78,7 +79,7 @@ describe('TemplateEntityV1alpah1', () => {
     await expect(policy.enforce(entity)).rejects.toThrow(/type/);
   });
 
-  it('acceptps any other type', async () => {
+  it('accepts any other type', async () => {
     (entity as any).spec.type = 'hallo';
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
@@ -87,4 +88,9 @@ describe('TemplateEntityV1alpah1', () => {
     (entity as any).spec.type = '';
     await expect(policy.enforce(entity)).rejects.toThrow(/type/);
   });
+
+  it('rejects missing templater', async () => {
+    (entity as any).spec.templater = '';
+    await expect(policy.enforce(entity)).rejects.toThrow(/templater/);
+  });
 });

packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts

@@ -26,6 +26,7 @@ export interface TemplateEntityV1alpha1 extends Entity {
   kind: typeof KIND;
   spec: {
     type: string;
+    templater: string;
     path?: string;
     schema: JSONSchema;
   };
@@ -43,6 +44,7 @@ export class TemplateEntityV1alpha1Policy implements EntityPolicy {
           type: yup.string().required().min(1),
           path: yup.string(),
           schema: yup.object().required(),
+          templater: yup.string().required(),
         })
         .required(),
     });

plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml

@@ -8,7 +8,7 @@ metadata:
     - Recommended
     - React
 spec:
-  processor: cookiecutter
+  templater: cookiecutter
   type: website
   path: '.'
   schema:

plugins/scaffolder-backend/sample-templates/springboot-template/template.yaml

@@ -8,7 +8,7 @@ metadata:
     - Recommended
     - Java
 spec:
-  processor: cookiecutter
+  templater: cookiecutter
   type: service
   path: '.'
   schema:

plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts

@@ -37,7 +37,8 @@ describe('JobProcessor', () => {
       generation: 1,
     },
     spec: {
-      type: 'cookiecutter',
+      type: 'website',
+      templater: 'cookiecutter',
       path: './template',
       schema: {
         $schema: 'http://json-schema.org/draft-07/schema#',

plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts

@@ -47,7 +47,8 @@ describe('GitHubPreparer', () => {
         generation: 1,
       },
       spec: {
-        type: 'cookiecutter',
+        type: 'website',
+        templater: 'cookiecutter',
         path: './template',
         schema: {
           $schema: 'http://json-schema.org/draft-07/schema#',

plugins/scaffolder-backend/src/scaffolder/stages/prepare/helpers.test.ts

@@ -39,7 +39,8 @@ describe('Helpers', () => {
           generation: 1,
         },
         spec: {
-          type: 'cookiecutter',
+          type: 'website',
+          templater: 'cookiecutter',
           path: './template',
           schema: {
             $schema: 'http://json-schema.org/draft-07/schema#',
@@ -86,7 +87,8 @@ describe('Helpers', () => {
           generation: 1,
         },
         spec: {
-          type: 'cookiecutter',
+          type: 'website',
+          templater: 'cookiecutter',
           path: './template',
           schema: {
             $schema: 'http://json-schema.org/draft-07/schema#',
@@ -131,7 +133,8 @@ describe('Helpers', () => {
           generation: 1,
         },
         spec: {
-          type: 'cookiecutter',
+          type: 'website',
+          templater: 'cookiecutter',
           path: './template',
           schema: {
             $schema: 'http://json-schema.org/draft-07/schema#',
@@ -177,7 +180,8 @@ describe('Helpers', () => {
           generation: 1,
         },
         spec: {
-          type: 'cookiecutter',
+          type: 'website',
+          templater: 'cookiecutter',
           path: './template',
           schema: {
             $schema: 'http://json-schema.org/draft-07/schema#',
@@ -221,7 +225,8 @@ describe('Helpers', () => {
           generation: 1,
         },
         spec: {
-          type: 'cookiecutter',
+          type: 'website',
+          templater: 'cookiecutter',
           path: './template',
           schema: {
             $schema: 'http://json-schema.org/draft-07/schema#',

plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts

@@ -24,7 +24,7 @@ describe('Preparers', () => {
     metadata: {
       annotations: {
         'backstage.io/managed-by-location':
-          'file:/Users/blam/dev/spotify/backstage/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml',
+          'file:/Users/bingo/spotify/backstage/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml',
       },
       name: 'react-ssr-template',
       title: 'React SSR Template',
@@ -35,8 +35,9 @@ describe('Preparers', () => {
       generation: 1,
     },
     spec: {
-      type: 'cookiecutter',
+      templater: 'cookiecutter',
       path: '.',
+      type: 'website',
       schema: {
         $schema: 'http://json-schema.org/draft-07/schema#',
         required: ['storePath', 'owner'],
@@ -88,7 +89,8 @@ describe('Preparers', () => {
         generation: 1,
       },
       spec: {
-        type: 'cookiecutter',
+        type: 'website',
+        templater: 'cookiecutter',
         path: '.',
         schema: {
           $schema: 'http://json-schema.org/draft-07/schema#',

plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts

@@ -14,3 +14,4 @@
  * limitations under the License.
  */
 export * from './github';
+export * from './types';

plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts

@@ -16,6 +16,8 @@
 import { Writable, PassThrough } from 'stream';
 import Docker from 'dockerode';
 import fs from 'fs';
+import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import { InputError } from '@backstage/backend-common';
 
 export type RunDockerContainerOptions = {
   imageName: string;
@@ -26,6 +28,20 @@ export type RunDockerContainerOptions = {
   dockerClient: Docker;
 };
 
+/**
+ * Gets the templater key to use for templating from the entity
+ * @param entity Template entity
+ */
+export const getTemplaterKey = (entity: TemplateEntityV1alpha1): string => {
+  const { templater } = entity.spec;
+
+  if (!templater) {
+    throw new InputError('Template does not have a required templating key');
+  }
+
+  return templater;
+};
+
 /**
  *
  * @param options the options object

plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts

@@ -16,3 +16,4 @@
 export * from './cookiecutter';
 export * from './types';
 export * from './helpers';
+export * from './templaters';

plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.test.ts

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Templaters } from '.';
+import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import { CookieCutter } from './cookiecutter';
+
+describe('Templaters', () => {
+  const mockTemplate: TemplateEntityV1alpha1 = {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Template',
+    metadata: {
+      annotations: {
+        'backstage.io/managed-by-location':
+          'file:/Users/bingo/spotify/backstage/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml',
+      },
+      name: 'react-ssr-template',
+      title: 'React SSR Template',
+      description:
+        'Next.js application skeleton for creating isomorphic web applications.',
+      uid: '7357f4c5-aa58-4a1e-9670-18931eef771f',
+      etag: 'YWUxZWQyY2EtZDkxMC00MDM0LWI0ODAtMDgwMWY0YzdlMWIw',
+      generation: 1,
+    },
+    spec: {
+      templater: 'cookiecutter',
+      path: '.',
+      type: 'website',
+      schema: {
+        $schema: 'http://json-schema.org/draft-07/schema#',
+        required: ['storePath', 'owner'],
+        properties: {
+          owner: {
+            type: 'string',
+            title: 'Owner',
+            description: 'Who is going to own this component',
+          },
+          storePath: {
+            type: 'string',
+            title: 'Store path',
+            description: 'GitHub store path in org/repo format',
+          },
+        },
+      },
+    },
+  };
+  it('should throw an error when the templater is not registered', () => {
+    const templaters = new Templaters();
+
+    expect(() => templaters.get(mockTemplate)).toThrow(
+      expect.objectContaining({
+        message: 'No templater registered for template: "cookiecutter"',
+      }),
+    );
+  });
+  it('should return the correct templater when the templater matches', () => {
+    const templaters = new Templaters();
+    const templater = new CookieCutter();
+
+    templaters.register('cookiecutter', templater);
+
+    expect(templaters.get(mockTemplate)).toBe(templater);
+  });
+
+  it('should throw an error if the templater does not exist in the entity', () => {
+    const brokenTemplate: TemplateEntityV1alpha1 = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Template',
+      metadata: {
+        annotations: {},
+        name: 'react-ssr-template',
+        title: 'React SSR Template',
+        description:
+          'Next.js application skeleton for creating isomorphic web applications.',
+        uid: '7357f4c5-aa58-4a1e-9670-18931eef771f',
+        etag: 'YWUxZWQyY2EtZDkxMC00MDM0LWI0ODAtMDgwMWY0YzdlMWIw',
+        generation: 1,
+      },
+      spec: {
+        type: 'website',
+        path: '.',
+        templater: '',
+        schema: {
+          $schema: 'http://json-schema.org/draft-07/schema#',
+          required: ['storePath', 'owner'],
+          properties: {
+            owner: {
+              type: 'string',
+              title: 'Owner',
+              description: 'Who is going to own this component',
+            },
+            storePath: {
+              type: 'string',
+              title: 'Store path',
+              description: 'GitHub store path in org/repo format',
+            },
+          },
+        },
+      },
+    };
+
+    const templaters = new Templaters();
+
+    expect(() => templaters.get(brokenTemplate)).toThrow(
+      expect.objectContaining({
+        name: 'InputError',
+        message: expect.stringContaining(
+          'Template does not have a required templating key',
+        ),
+      }),
+    );
+  });
+});

plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  TemplaterBase,
+  SupportedTemplatingKey,
+  TemplaterBuilder,
+} from './types';
+
+import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import { getTemplaterKey } from './helpers';
+
+export class Templaters implements TemplaterBuilder {
+  private preparerMap = new Map<SupportedTemplatingKey, TemplaterBase>();
+
+  register(templaterKey: SupportedTemplatingKey, templater: TemplaterBase) {
+    this.preparerMap.set(templaterKey, templater);
+  }
+
+  get(template: TemplateEntityV1alpha1): TemplaterBase {
+    const templaterKey = getTemplaterKey(template);
+    const preparer = this.preparerMap.get(templaterKey);
+
+    if (!preparer) {
+      throw new Error(
+        `No templater registered for template: "${templaterKey}"`,
+      );
+    }
+
+    return preparer;
+  }
+}

plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts

@@ -13,10 +13,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 import type { Writable } from 'stream';
 import Docker from 'dockerode';
 import { JsonValue } from '@backstage/config';
+import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 
 /**
  * Currently the required template values. The owner
@@ -55,3 +55,16 @@ export type TemplaterBase = {
 export type TemplaterConfig = {
   templater?: TemplaterBase;
 };
+
+/**
+ * List of supported templating options
+ */
+export type SupportedTemplatingKey = 'cookiecutter' | string;
+
+/**
+ * The templater builder holds the templaters ready for run time
+ */
+export type TemplaterBuilder = {
+  register(protocol: SupportedTemplatingKey, templater: TemplaterBase): void;
+  get(template: TemplateEntityV1alpha1): TemplaterBase;
+};

plugins/scaffolder-backend/src/service/router.ts

@@ -16,23 +16,24 @@
 
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { JsonValue } from '@backstage/config';
-import { Octokit } from '@octokit/rest';
 import Docker from 'dockerode';
 import express from 'express';
 import Router from 'express-promise-router';
 import { Logger } from 'winston';
 import {
-  GithubPublisher,
   JobProcessor,
   PreparerBuilder,
   RequiredTemplateValues,
   StageContext,
-  TemplaterBase,
+  TemplaterBuilder,
+  Publisher,
 } from '../scaffolder';
 
 export interface RouterOptions {
   preparers: PreparerBuilder;
-  templater: TemplaterBase;
+  templaters: TemplaterBuilder;
+  publisher: Publisher;
+
   logger: Logger;
   dockerClient: Docker;
 }
@@ -42,9 +43,14 @@ export async function createRouter(
 ): Promise<express.Router> {
   const router = Router();
 
-  const githubClient = new Octokit({ auth: process.env.GITHUB_ACCESS_TOKEN });
-  const { preparers, templater, logger: parentLogger, dockerClient } = options;
-  const githubPulisher = new GithubPublisher({ client: githubClient });
+  const {
+    preparers,
+    templaters,
+    publisher,
+    logger: parentLogger,
+    dockerClient,
+  } = options;
+
   const logger = parentLogger.child({ plugin: 'scaffolder' });
   const jobProcessor = new JobProcessor();
 
@@ -106,6 +112,7 @@ export async function createRouter(
           {
             name: 'Run the templater',
             handler: async (ctx: StageContext<{ skeletonDir: string }>) => {
+              const templater = templaters.get(ctx.entity);
               const { resultDir } = await templater.run({
                 directory: ctx.skeletonDir,
                 dockerClient,
@@ -120,7 +127,8 @@ export async function createRouter(
             name: 'Publish template',
             handler: async (ctx: StageContext<{ resultDir: string }>) => {
               ctx.logger.info('Should not store the template');
-              const { remoteUrl } = await githubPulisher.publish({
+              const { remoteUrl } = await publisher.publish({
+                entity: ctx.entity,
                 values: ctx.values,
                 directory: ctx.resultDir,
               });