Merge pull request #1437 from spotify/mob/job-processor

Job Processor and Scaffolder Flow Implementation

plugins/scaffolder-backend/package.json

@@ -37,6 +37,7 @@
     "helmet": "^3.22.0",
     "morgan": "^1.10.0",
     "nodegit": "0.26.5",
+    "uuid": "^8.2.0",
     "winston": "^3.2.1"
   },
   "devDependencies": {

plugins/scaffolder-backend/src/scaffolder/index.ts

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-export * from './templater';
-export * from './prepare';
-export * from './templater/cookiecutter';
+export * from './stages/templater';
+export * from './stages/templater/cookiecutter';
+export * from './stages/prepare';
+export * from './jobs';

plugins/scaffolder-backend/src/scaffolder/jobs/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export * from './processor';
+export * from './types';

plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { PassThrough } from 'stream';
+import winston from 'winston';
+import { JsonValue } from '@backstage/config';
+
+export const useLogStream = (meta: Record<string, JsonValue>) => {
+  const log: string[] = [];
+
+  // Create an empty stream to collect all the log lines into
+  // one variable for the API.
+  const stream = new PassThrough();
+  stream.on('data', chunk => log.push(chunk.toString()));
+
+  const logger = winston.createLogger({
+    level: process.env.LOG_LEVEL || 'info',
+    format: winston.format.combine(
+      winston.format.colorize(),
+      winston.format.timestamp(),
+      winston.format.simple(),
+    ),
+    defaultMeta: meta,
+  });
+
+  logger.add(new winston.transports.Stream({ stream }));
+
+  return {
+    log,
+    stream,
+    logger,
+  };
+};

plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts

@@ -0,0 +1,278 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { JobProcessor } from './processor';
+import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import { StageInput } from './types';
+
+describe('JobProcessor', () => {
+  const mockEntity: TemplateEntityV1alpha1 = {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Template',
+    metadata: {
+      annotations: {
+        'backstage.io/managed-by-location':
+          'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
+      },
+      name: 'graphql-starter',
+      title: 'GraphQL Service',
+      description:
+        'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
+      uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
+      etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
+
+      generation: 1,
+    },
+    spec: {
+      type: 'cookiecutter',
+      path: './template',
+    },
+  };
+
+  const mockValues = { component_id: 'bob' };
+
+  describe('create', () => {
+    it('creates should create a new job with a unique id', async () => {
+      const processor = new JobProcessor();
+
+      const job = processor.create({
+        entity: mockEntity,
+        values: mockValues,
+        stages: [],
+      });
+
+      expect(job.id).toMatch(
+        /^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i,
+      );
+    });
+
+    it('should setup the correct context for the job', async () => {
+      const processor = new JobProcessor();
+
+      const job = processor.create({
+        entity: mockEntity,
+        values: mockValues,
+        stages: [],
+      });
+
+      expect(job.context.entity).toBe(mockEntity);
+      expect(job.context.values).toBe(mockValues);
+    });
+
+    it('should set the status as pending', async () => {
+      const processor = new JobProcessor();
+
+      const job = processor.create({
+        entity: mockEntity,
+        values: mockValues,
+        stages: [],
+      });
+
+      expect(job.status).toBe('PENDING');
+    });
+
+    it('should create the correct stages', async () => {
+      const stages: StageInput[] = [
+        {
+          name: 'Do something cool step 1',
+          handler: jest.fn(),
+        },
+        {
+          name: 'Do something cool step 2',
+          handler: jest.fn(),
+        },
+      ];
+
+      const processor = new JobProcessor();
+
+      const job = processor.create({
+        entity: mockEntity,
+        values: mockValues,
+        stages,
+      });
+
+      expect(job.stages).toHaveLength(stages.length);
+
+      for (let i = 0; i < job.stages.length; i++) {
+        expect(job.stages[i].name).toBe(stages[i].name);
+        expect(job.stages[i].status).toBe('PENDING');
+      }
+    });
+  });
+
+  describe('get', () => {
+    it('return undefined for when the job does not exist', () => {
+      const processor = new JobProcessor();
+      expect(processor.get('123')).not.toBeDefined();
+    });
+
+    it('should return the exact same instance of the job when one is created', async () => {
+      const processor = new JobProcessor();
+      const job = processor.create({
+        entity: mockEntity,
+        values: mockValues,
+        stages: [],
+      });
+
+      expect(processor.get(job.id)).toBe(job);
+    });
+  });
+  describe('process', () => {
+    it('throws an error when the status of the job is not in pending state', async () => {
+      const processor = new JobProcessor();
+      const job = processor.create({
+        entity: mockEntity,
+        values: mockValues,
+        stages: [],
+      });
+
+      job.status = 'STARTED';
+
+      await expect(processor.run(job)).rejects.toThrow(
+        /Job is not in a 'PENDING' state/,
+      );
+    });
+
+    it('will call each of the handlers in the stages', async () => {
+      const stages: StageInput[] = [
+        {
+          name: 'c/o',
+          handler: jest.fn(),
+        },
+        {
+          name: 'g/p',
+          handler: jest.fn(),
+        },
+      ];
+
+      const processor = new JobProcessor();
+      const job = processor.create({
+        entity: mockEntity,
+        values: mockValues,
+        stages,
+      });
+
+      await processor.run(job);
+
+      for (const stage of stages) {
+        expect(stage.handler).toHaveBeenCalled();
+      }
+    });
+
+    it('should set all stages to complete and the job to complete when finishes without errors', async () => {
+      const stages: StageInput[] = [
+        {
+          name: 'c/o',
+          handler: jest.fn(),
+        },
+        {
+          name: 'g/p',
+          handler: jest.fn(),
+        },
+      ];
+
+      const processor = new JobProcessor();
+      const job = processor.create({
+        entity: mockEntity,
+        values: mockValues,
+        stages,
+      });
+
+      await processor.run(job);
+
+      for (const stage of job.stages) {
+        expect(stage.status).toBe('COMPLETED');
+      }
+
+      expect(job.status).toBe('COMPLETED');
+    });
+
+    it('should merge the return value from previous steps into the context of the next step', async () => {
+      const stages: StageInput[] = [
+        {
+          name: 'c/o',
+          handler: jest
+            .fn()
+            .mockResolvedValue({ first: 'ben', second: 'lambert' }),
+        },
+        {
+          name: 'g/p',
+          handler: jest
+            .fn()
+            .mockResolvedValue({ second: 'linus', third: 'lambert' }),
+        },
+        {
+          name: 'go',
+          handler: jest.fn(),
+        },
+      ];
+
+      const processor = new JobProcessor();
+      const job = processor.create({
+        entity: mockEntity,
+        values: mockValues,
+        stages,
+      });
+
+      await processor.run(job);
+
+      expect(stages[1].handler).toHaveBeenCalledWith(
+        expect.objectContaining({ first: 'ben', second: 'lambert' }),
+      );
+
+      expect(stages[2].handler).toHaveBeenCalledWith(
+        expect.objectContaining({
+          first: 'ben',
+          second: 'linus',
+          third: 'lambert',
+        }),
+      );
+    });
+
+    it('should fail the job and the step if one of them fails', async () => {
+      const fail = new Error('something went wrong here');
+      const stages: StageInput[] = [
+        {
+          name: 'c/o',
+          handler: jest.fn(),
+        },
+        {
+          name: 'g/p',
+          handler: jest.fn().mockRejectedValue(fail),
+        },
+        {
+          name: 'go',
+          handler: jest.fn(),
+        },
+      ];
+
+      const processor = new JobProcessor();
+      const job = processor.create({
+        entity: mockEntity,
+        values: mockValues,
+        stages,
+      });
+
+      await processor.run(job);
+
+      expect(job.status).toBe('FAILED');
+      expect(job.stages[0].status).toBe('COMPLETED');
+      expect(job.stages[1].status).toBe('FAILED');
+      expect(job.stages[2].status).toBe('PENDING');
+      expect(job.error?.message).toBe('something went wrong here');
+      expect(job.stages[1].log.join()).toContain('something went wrong here');
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts

@@ -0,0 +1,139 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Processor, Job, StageContext, StageInput } from './types';
+import { JsonValue } from '@backstage/config';
+import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import * as uuid from 'uuid';
+import Docker from 'dockerode';
+import { RequiredTemplateValues, TemplaterBase } from '../stages/templater';
+import { PreparerBuilder } from '../stages/prepare';
+import { useLogStream } from './logger';
+
+export type JobProcessorArguments = {
+  preparers: PreparerBuilder;
+  templater: TemplaterBase;
+  dockerClient: Docker;
+};
+
+export type JobAndDirectoryTuple = {
+  job: Job;
+  directory: string;
+};
+
+export class JobProcessor implements Processor {
+  private jobs = new Map<string, Job>();
+
+  create({
+    entity,
+    values,
+    stages,
+  }: {
+    entity: TemplateEntityV1alpha1;
+    values: RequiredTemplateValues & Record<string, JsonValue>;
+    stages: StageInput[];
+  }): Job {
+    const id = uuid.v4();
+    const { logger, stream } = useLogStream({ id });
+
+    const context: StageContext = {
+      entity,
+      values,
+      logger,
+      logStream: stream,
+    };
+
+    const job: Job = {
+      id,
+      context,
+      stages: stages.map(stage => ({
+        handler: stage.handler,
+        log: [],
+        name: stage.name,
+        status: 'PENDING',
+      })),
+      status: 'PENDING',
+    };
+
+    this.jobs.set(job.id, job);
+
+    return job;
+  }
+
+  get(id: string): Job | undefined {
+    return this.jobs.get(id);
+  }
+
+  async run(job: Job): Promise<void> {
+    if (job.status !== 'PENDING') {
+      throw new Error("Job is not in a 'PENDING' state");
+    }
+
+    job.status = 'STARTED';
+
+    try {
+      for (const stage of job.stages) {
+        // Create a logger for each stage so we can create seperate
+        // Streams for each step.
+        const { logger, log, stream } = useLogStream({
+          id: job.id,
+          stage: stage.name,
+        });
+        // Attach the logger to the stage, and setup some timestamps.
+        stage.log = log;
+        stage.startedAt = Date.now();
+
+        try {
+          // Run the handler with the context created for the Job and some
+          // Additional logging helpers.
+          const handlerResponse = await stage.handler({
+            ...job.context,
+            logger,
+            logStream: stream,
+          });
+
+          // If the handler returns something, then let's merge this onto the ontext
+          // For the next stage to use as it might be relevant.
+          if (handlerResponse) {
+            job.context = {
+              ...job.context,
+              ...handlerResponse,
+            };
+          }
+
+          // Complete the current stage
+          stage.status = 'COMPLETED';
+        } catch (error) {
+          // Log to the current stage the error that occured and fail the stage.
+          logger.error(`Stage failed with error: ${error.message}`);
+          stage.status = 'FAILED';
+
+          // Throw the error so the job can be failed too.
+          throw error;
+        } finally {
+          // Always set the stage end timestamp.
+          stage.endedAt = Date.now();
+        }
+      }
+
+      // If all went to plan, complete the job.
+      job.status = 'COMPLETED';
+    } catch (error) {
+      // If something went wrong, fail the job, and set the error property on the job.
+      job.error = { name: error.name, message: error.message };
+      job.status = 'FAILED';
+    }
+  }
+}

plugins/scaffolder-backend/src/scaffolder/jobs/types.ts

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type { Writable } from 'stream';
+import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import { JsonValue } from '@backstage/config';
+import { RequiredTemplateValues } from '../stages/templater';
+import { Logger } from 'winston';
+
+// Context will be a mutable object which is passed between stages
+// To share data, but also thinking that we can pass in functions here too
+// To maybe create sub steps or fail the entire thing, or skip stages down the line.
+export type StageContext<T = {}> = {
+  values: RequiredTemplateValues & Record<string, JsonValue>;
+  entity: TemplateEntityV1alpha1;
+  logger: Logger;
+  logStream: Writable;
+} & T;
+
+export type ProcessorStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED';
+
+export interface Stage extends StageInput {
+  log: string[];
+  status: ProcessorStatus;
+  startedAt?: number;
+  endedAt?: number;
+}
+
+export interface StageInput<T = {}> {
+  name: string;
+  handler(ctx: StageContext<T>): Promise<void | object>;
+}
+
+export type Job = {
+  id: string;
+  context: StageContext;
+  status: ProcessorStatus;
+  stages: Stage[];
+  error?: Error;
+};
+
+export type Processor = {
+  create({
+    entity,
+    values,
+    stages,
+  }: {
+    entity: TemplateEntityV1alpha1;
+    values: RequiredTemplateValues & Record<string, JsonValue>;
+    stages: StageInput[];
+  }): Job;
+
+  get(id: string): Job | undefined;
+
+  run(job: Job): Promise<void>;
+};

plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts

@@ -60,7 +60,7 @@ describe('GitHubPreparer', () => {
       1,
       'https://github.com/benjdlambert/backstage-graphql-template',
       expect.any(String),
-      { checkoutOpts: { paths: ['template'] } },
+      {},
     );
   });
   it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
@@ -71,7 +71,7 @@ describe('GitHubPreparer', () => {
       1,
       'https://github.com/benjdlambert/backstage-graphql-template',
       expect.any(String),
-      { checkoutOpts: {} },
+      {},
     );
   });
 

plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts

@@ -21,7 +21,7 @@ import { parseLocationAnnotation } from './helpers';
 import { InputError } from '@backstage/backend-common';
 import { PreparerBase } from './types';
 import GitUriParser from 'git-url-parse';
-import { Clone, CheckoutOptions } from 'nodegit';
+import { Clone } from 'nodegit';
 
 export class GithubPreparer implements PreparerBase {
   async prepare(template: TemplateEntityV1alpha1): Promise<string> {
@@ -45,13 +45,7 @@ export class GithubPreparer implements PreparerBase {
       template.spec.path ?? '.',
     );
 
-    const checkoutOptions = new CheckoutOptions();
-    if (template.spec.path) {
-      checkoutOptions.paths = [templateDirectory];
-    }
-
     await Clone.clone(repositoryCheckoutUrl, tempDir, {
-      checkoutOpts: checkoutOptions,
       // TODO(blam): Maybe need some auth here?
     });
 

plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 import type { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import { Logger } from 'winston';
 
 export type PreparerBase = {
   /**
@@ -21,7 +22,10 @@ export type PreparerBase = {
    * with contents from the remote location in temporary storage and return the path
    * @param template The template entity from the Service Catalog
    */
-  prepare(template: TemplateEntityV1alpha1): Promise<string>;
+  prepare(
+    template: TemplateEntityV1alpha1,
+    opts: { logger: Logger },
+  ): Promise<string>;
 };
 
 export type PreparerBuilder = {

plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts

@@ -18,6 +18,7 @@ jest.mock('./helpers', () => ({ runDockerContainer: jest.fn() }));
 import { CookieCutter } from './cookiecutter';
 import fs from 'fs-extra';
 import os from 'os';
+import path from 'path';
 import { RunDockerContainerOptions } from './helpers';
 import { PassThrough } from 'stream';
 import Docker from 'dockerode';
@@ -33,12 +34,15 @@ describe('CookieCutter Templater', () => {
 
   beforeEach(async () => {
     jest.clearAllMocks();
-
-    await fs.remove(`${os.tmpdir()}/cookiecutter.json`);
   });
 
+  const mkTemp = async () => {
+    const tempDir = os.tmpdir();
+    return await fs.promises.mkdtemp(path.join(tempDir, 'temp'));
+  };
+
   it('should write a cookiecutter.json file with the values from the entitiy', async () => {
-    const tempdir = os.tmpdir();
+    const tempdir = await mkTemp();
 
     const values = {
       component_id: 'test',
@@ -53,10 +57,11 @@ describe('CookieCutter Templater', () => {
   });
 
   it('should merge any value that is in the cookiecutter.json path already', async () => {
-    const tempdir = os.tmpdir();
+    const tempdir = await mkTemp();
     const existingJson = {
       _copy_without_render: ['./github/workflows/*'],
     };
+
     await fs.writeJSON(`${tempdir}/cookiecutter.json`, existingJson);
 
     const values = {
@@ -72,7 +77,7 @@ describe('CookieCutter Templater', () => {
   });
 
   it('should throw an error if the cookiecutter json is malformed and not missing', async () => {
-    const tempdir = os.tmpdir();
+    const tempdir = await mkTemp();
 
     await fs.writeFile(`${tempdir}/cookiecutter.json`, "{'");
 
@@ -87,7 +92,7 @@ describe('CookieCutter Templater', () => {
   });
 
   it('should run the correct docker container with the correct bindings for the volumes', async () => {
-    const tempdir = os.tmpdir();
+    const tempdir = await mkTemp();
 
     const values = {
       component_id: 'test',
@@ -97,35 +102,43 @@ describe('CookieCutter Templater', () => {
     await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
 
     expect(runDockerContainer).toHaveBeenCalledWith({
-      imageName: 'backstage/cookiecutter',
-      args: ['cookiecutter', '--no-input', '-o', '/result', '/template'],
+      imageName: 'spotify/backstage-cookiecutter',
+      args: [
+        'cookiecutter',
+        '--no-input',
+        '-o',
+        '/result',
+        '/template',
+        '--verbose',
+      ],
       templateDir: tempdir,
-      resultDir: `${tempdir}/result`,
+      resultDir: expect.stringContaining(`${tempdir}-result`),
       logStream: undefined,
       dockerClient: mockDocker,
     });
   });
+
   it('should return the result path to the end templated folder', async () => {
-    const tempdir = os.tmpdir();
+    const tempdir = await mkTemp();
 
     const values = {
       component_id: 'test',
       description: 'description',
     };
 
-    const path = await cookie.run({
+    const returnPath = await cookie.run({
       directory: tempdir,
       values,
       dockerClient: mockDocker,
     });
 
-    expect(path).toBe(`${tempdir}/result`);
+    expect(returnPath.startsWith(`${tempdir}-result`)).toBeTruthy();
   });
 
   it('should pass through the streamer to the run docker helper', async () => {
     const stream = new PassThrough();
 
-    const tempdir = os.tmpdir();
+    const tempdir = await mkTemp();
 
     const values = {
       component_id: 'test',
@@ -140,10 +153,17 @@ describe('CookieCutter Templater', () => {
     });
 
     expect(runDockerContainer).toHaveBeenCalledWith({
-      imageName: 'backstage/cookiecutter',
-      args: ['cookiecutter', '--no-input', '-o', '/result', '/template'],
+      imageName: 'spotify/backstage-cookiecutter',
+      args: [
+        'cookiecutter',
+        '--no-input',
+        '-o',
+        '/result',
+        '/template',
+        '--verbose',
+      ],
       templateDir: tempdir,
-      resultDir: `${tempdir}/result`,
+      resultDir: expect.stringContaining(`${tempdir}-result`),
       logStream: stream,
       dockerClient: mockDocker,
     });

plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts

@@ -48,14 +48,18 @@ export class CookieCutter implements TemplaterBase {
     await fs.writeJSON(`${options.directory}/cookiecutter.json`, cookieInfo);
 
     const templateDir = options.directory;
-
-    // TODO(blam): This should be an entirely different directory on the host machine
-    // not in the template directory
-    const resultDir = `${templateDir}/result`;
+    const resultDir = await fs.promises.mkdtemp(`${options.directory}-result`);
 
     await runDockerContainer({
-      imageName: 'backstage/cookiecutter',
-      args: ['cookiecutter', '--no-input', '-o', '/result', '/template'],
+      imageName: 'spotify/backstage-cookiecutter',
+      args: [
+        'cookiecutter',
+        '--no-input',
+        '-o',
+        '/result',
+        '/template',
+        '--verbose',
+      ],
       templateDir,
       resultDir,
       logStream: options.logStream,

plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts

@@ -16,6 +16,7 @@
 
 import type { Writable } from 'stream';
 import Docker from 'dockerode';
+import { JsonValue } from '@backstage/config';
 
 export interface RequiredTemplateValues {
   component_id: string;
@@ -23,7 +24,7 @@ export interface RequiredTemplateValues {
 
 export interface TemplaterRunOptions {
   directory: string;
-  values: RequiredTemplateValues & object;
+  values: RequiredTemplateValues & Record<string, JsonValue>;
   logStream?: Writable;
   dockerClient: Docker;
 }

plugins/scaffolder-backend/src/service/router.ts

@@ -17,10 +17,11 @@
 import { Logger } from 'winston';
 import Router from 'express-promise-router';
 import express from 'express';
-import { PreparerBuilder, TemplaterBase } from '../scaffolder';
+import { PreparerBuilder, TemplaterBase, JobProcessor } from '../scaffolder';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import Docker from 'dockerode';
-
+import {} from '@backstage/backend-common';
+import { StageContext } from '../scaffolder/jobs/types';
 export interface RouterOptions {
   preparers: PreparerBuilder;
   templater: TemplaterBase;
@@ -35,51 +36,117 @@ export async function createRouter(
   const { preparers, templater, logger: parentLogger, dockerClient } = options;
   const logger = parentLogger.child({ plugin: 'scaffolder' });
 
-  router.post('/v1/jobs', async (_, res) => {
-    // TODO(blam): Create a unique job here and return the ID so that
-    // The end user can poll for updates on the current job
-    res.status(201).json({ accepted: true });
+  const jobProcessor = new JobProcessor();
 
-    // TODO(blam): Take this entity from the post body sent from the frontend
-    const mockEntity: TemplateEntityV1alpha1 = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'Template',
-      metadata: {
-        annotations: {
-          'backstage.io/managed-by-location':
-            'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
+  router
+    .get('/v1/job/:jobId/stage/:index/log', ({ params }, res) => {
+      const job = jobProcessor.get(params.jobId);
+
+      if (!job) {
+        res.status(404).send({ error: 'job not found' });
+        return;
+      }
+
+      res.send(job.stages[Number(params.index)].log.join(''));
+    })
+    .get('/v1/job/:jobId', ({ params }, res) => {
+      const job = jobProcessor.get(params.jobId);
+
+      if (!job) {
+        res.status(404).send({ error: 'job not found' });
+        return;
+      }
+
+      res.send({
+        id: job.id,
+        metadata: {
+          ...job.context,
+          logger: undefined,
+          logStream: undefined,
         },
-        name: 'graphql-starter',
-        title: 'GraphQL Service',
-        description:
-          'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
-        uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
-        etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
+        status: job.status,
+        stages: job.stages.map(stage => ({
+          ...stage,
+          handler: undefined,
+        })),
+        error: job.error,
+      });
+    })
+    .post('/v1/jobs', async (_, res) => {
+      // TODO(blam): Create a unique job here and return the ID so that
+      // The end user can poll for updates on the current job
 
-        generation: 1,
-      },
-      spec: {
-        type: 'cookiecutter',
-        path: './template',
-      },
-    };
+      // TODO(blam): Take this entity from the post body sent from the frontend
+      const mockEntity: TemplateEntityV1alpha1 = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Template',
+        metadata: {
+          annotations: {
+            'backstage.io/managed-by-location':
+              'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
+          },
+          name: 'graphql-starter',
+          title: 'GraphQL Service',
+          description:
+            'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
+          uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
+          etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
 
-    // Get the preparer for the mock entity
-    const preparer = preparers.get(mockEntity);
+          generation: 1,
+        },
+        spec: {
+          type: 'cookiecutter',
+          path: './template',
+        },
+      };
 
-    // Run the preparer for the mock entity to produce a temporary directory with template in
-    const skeletonPath = await preparer.prepare(mockEntity);
+      const job = jobProcessor.create({
+        entity: mockEntity,
+        values: { component_id: 'blob' },
+        stages: [
+          {
+            name: 'Prepare the skeleton',
+            handler: async ctx => {
+              const preparer = preparers.get(ctx.entity);
+              const skeletonDir = await preparer.prepare(ctx.entity, {
+                logger: ctx.logger,
+              });
+              return { skeletonDir };
+            },
+          },
+          {
+            name: 'Run the templater',
+            handler: async (ctx: StageContext<{ skeletonDir: string }>) => {
+              const resultDir = await templater.run({
+                directory: ctx.skeletonDir,
+                dockerClient,
+                logStream: ctx.logStream,
+                values: ctx.values,
+              });
 
-    // Run the templater on the mock directory with values from the post body
-    const templatedPath = await templater.run({
-      directory: skeletonPath,
-      values: { component_id: 'test' },
-      dockerClient,
+              return { resultDir };
+            },
+          },
+          {
+            name: 'Create VCS Repo',
+            handler: async (ctx: StageContext<{ resultDir: string }>) => {
+              ctx.logger.info('Should now create the VCS repo');
+            },
+          },
+          {
+            name: 'Push to remote',
+            handler: async ctx => {
+              ctx.logger.info('Should now push to the remote');
+            },
+          },
+        ],
+      });
+
+      res.status(201).json({ id: job.id });
+
+      jobProcessor.run(job);
     });
 
-    console.warn(templatedPath);
-  });
-
   const app = express();
   app.set('logger', logger);
   app.use('/', router);