Merge branch 'mob/create-vcs-step' of github.com:spotify/backstage into shmidt-i/scaffolder-flow-frontend

plugins/scaffolder-backend/scripts/mock-data

@@ -9,4 +9,5 @@ for URL in \
     --request POST 'localhost:7000/catalog/locations' \
     --header 'Content-Type: application/json' \
     --data-raw "{\"type\": \"file\", \"target\": \"$(pwd)/sample-templates/${URL}/template.yaml\"}"
+  echo
 done

plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts

@@ -17,7 +17,7 @@ import { PassThrough } from 'stream';
 import winston from 'winston';
 import { JsonValue } from '@backstage/config';
 
-export const useLogStream = (meta: Record<string, JsonValue>) => {
+export const makeLogStream = (meta: Record<string, JsonValue>) => {
   const log: string[] = [];
 
   // Create an empty stream to collect all the log lines into

plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts

@@ -16,6 +16,7 @@
 import { JobProcessor } from './processor';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { StageInput } from './types';
+import { RequiredTemplateValues } from '../stages/templater';
 
 describe('JobProcessor', () => {
   const mockEntity: TemplateEntityV1alpha1 = {
@@ -41,7 +42,10 @@ describe('JobProcessor', () => {
     },
   };
 
-  const mockValues = { component_id: 'bob' };
+  const mockValues: RequiredTemplateValues = {
+    owner: 'blobby',
+    storePath: 'spotify/mock-repo',
+  };
 
   describe('create', () => {
     it('creates should create a new job with a unique id', async () => {

plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts

@@ -20,7 +20,7 @@ import * as uuid from 'uuid';
 import Docker from 'dockerode';
 import { RequiredTemplateValues, TemplaterBase } from '../stages/templater';
 import { PreparerBuilder } from '../stages/prepare';
-import { useLogStream } from './logger';
+import { makeLogStream } from './logger';
 
 export type JobProcessorArguments = {
   preparers: PreparerBuilder;
@@ -46,7 +46,7 @@ export class JobProcessor implements Processor {
     stages: StageInput[];
   }): Job {
     const id = uuid.v4();
-    const { logger, stream } = useLogStream({ id });
+    const { logger, stream } = makeLogStream({ id });
 
     const context: StageContext = {
       entity,
@@ -87,7 +87,7 @@ export class JobProcessor implements Processor {
       for (const stage of job.stages) {
         // Create a logger for each stage so we can create seperate
         // Streams for each step.
-        const { logger, log, stream } = useLogStream({
+        const { logger, log, stream } = makeLogStream({
           id: job.id,
           stage: stage.name,
         });

plugins/scaffolder-backend/src/scaffolder/jobs/types.ts

@@ -31,7 +31,7 @@ export type StageContext<T = {}> = {
 
 export type ProcessorStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED';
 
-export interface Stage extends StageInput {
+export interface StageResult extends StageInput {
   log: string[];
   status: ProcessorStatus;
   startedAt?: number;
@@ -47,7 +47,7 @@ export type Job = {
   id: string;
   context: StageContext;
   status: ProcessorStatus;
-  stages: Stage[];
+  stages: StageResult[];
   error?: Error;
 };
 

plugins/scaffolder-backend/src/scaffolder/stages/store/github.ts

@@ -33,15 +33,15 @@ export class GithubStorer implements Storer {
     entity: TemplateEntityV1alpha1;
     values: RequiredTemplateValues & Record<string, JsonValue>;
   }) {
+    const [owner, name] = values.storePath.split('/');
+
     const {
       data: { clone_url: cloneUrl },
     } = await this.client.repos.createInOrg({
-      name: values.component_id,
-      org: values.org as string,
+      name,
+      org: owner,
     });
 
-    console.warn(cloneUrl);
-
     return cloneUrl;
   }
 
@@ -53,8 +53,8 @@ export class GithubStorer implements Storer {
     const oid = await index.writeTree();
     await repo.createCommit(
       'HEAD',
-      Signature.now('Foo bar', 'foo@bar.com'),
-      Signature.now('Foo bar', 'foo@bar.com'),
+      Signature.now('Scaffolder', 'scaffolder@backstage.io'),
+      Signature.now('Scaffolder', 'scaffolder@backstage.io'),
       'initial commit',
       oid,
       [],

plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts

@@ -45,8 +45,10 @@ describe('CookieCutter Templater', () => {
     const tempdir = await mkTemp();
 
     const values = {
-      component_id: 'test',
+      owner: 'blobby',
+      storePath: 'spotify/end-repo',
       description: 'description',
+      component_id: 'newthing',
     };
 
     await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
@@ -65,8 +67,9 @@ describe('CookieCutter Templater', () => {
     await fs.writeJSON(`${tempdir}/cookiecutter.json`, existingJson);
 
     const values = {
-      component_id: 'hello',
-      description: 'im something cool',
+      owner: 'blobby',
+      storePath: 'spotify/end-repo',
+      component_id: 'something',
     };
 
     await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
@@ -82,8 +85,8 @@ describe('CookieCutter Templater', () => {
     await fs.writeFile(`${tempdir}/cookiecutter.json`, "{'");
 
     const values = {
-      component_id: 'hello',
-      description: 'im something cool',
+      owner: 'blobby',
+      storePath: 'spotify/end-repo',
     };
 
     await expect(
@@ -95,8 +98,9 @@ describe('CookieCutter Templater', () => {
     const tempdir = await mkTemp();
 
     const values = {
-      component_id: 'test',
-      description: 'description',
+      owner: 'blobby',
+      storePath: 'spotify/end-repo',
+      component_id: 'newthing',
     };
 
     await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
@@ -122,8 +126,9 @@ describe('CookieCutter Templater', () => {
     const tempdir = await mkTemp();
 
     const values = {
-      component_id: 'test',
-      description: 'description',
+      owner: 'blobby',
+      storePath: 'spotify/end-repo',
+      component_id: 'newthing',
     };
 
     const returnPath = await cookie.run({
@@ -141,8 +146,9 @@ describe('CookieCutter Templater', () => {
     const tempdir = await mkTemp();
 
     const values = {
-      component_id: 'test',
-      description: 'description',
+      owner: 'blobby',
+      storePath: 'spotify/end-repo',
+      component_id: 'newthing',
     };
 
     await cookie.run({

plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts

@@ -65,6 +65,6 @@ export class CookieCutter implements TemplaterBase {
       dockerClient: options.dockerClient,
     });
 
-    return path.resolve(resultDir, options.values.component_id);
+    return path.resolve(resultDir, options.values.component_id as string);
   }
 }

plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts

@@ -19,7 +19,8 @@ import Docker from 'dockerode';
 import { JsonValue } from '@backstage/config';
 
 export type RequiredTemplateValues = {
-  component_id: string;
+  owner: string;
+  storePath: string;
 };
 
 export type TemplaterRunOptions = {

plugins/scaffolder-backend/src/service/router.ts

@@ -17,13 +17,19 @@
 import { Logger } from 'winston';
 import Router from 'express-promise-router';
 import express from 'express';
-import { PreparerBuilder, TemplaterBase, JobProcessor } from '../scaffolder';
+import {
+  PreparerBuilder,
+  TemplaterBase,
+  JobProcessor,
+  RequiredTemplateValues,
+} from '../scaffolder';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import Docker from 'dockerode';
 import { InputError } from '@backstage/backend-common';
 import { StageContext } from '../scaffolder/jobs/types';
 import { Octokit } from '@octokit/rest';
 import { GithubStorer } from '../scaffolder/stages/store/github';
+import { JsonValue } from '@backstage/config';
 export interface RouterOptions {
   preparers: PreparerBuilder;
   templater: TemplaterBase;
@@ -51,7 +57,9 @@ export async function createRouter(
         return;
       }
 
-      res.send(job.stages[Number(params.index)].log.join(''));
+      const { log } = job.stages[Number(params.index)] ?? { log: [] };
+
+      res.send(log.join(''));
     })
     .get('/v1/job/:jobId', ({ params }, res) => {
       const job = jobProcessor.get(params.jobId);
@@ -77,27 +85,13 @@ export async function createRouter(
       });
     })
     .post('/v1/jobs', async (req, res) => {
-      // TODO(blam): Create a unique job here and return the ID so that
-      // The end user can poll for updates on the current job
-
-      // TODO(blam): Take this entity from the post body sent from the frontend
-      console.log(req.body);
       const template: TemplateEntityV1alpha1 = req.body.template;
-
-      if (!template) {
-        throw new InputError(
-          'You should specify the template to scaffold from',
-        );
-      }
+      const values: RequiredTemplateValues & Record<string, JsonValue> =
+        req.body.values;
 
       const job = jobProcessor.create({
         entity: template,
-        values: {
-          component_id: `blob${Date.now()}`,
-          org: 'shmidt-i-test',
-          description: 'test',
-          owner: 'somebody',
-        },
+        values,
         stages: [
           {
             name: 'Prepare the skeleton',