feat(scaffolder): finally managed to get typescript to play nice with the new stage approach

packages/backend-common/src/logging/rootLogger.ts

@@ -13,31 +13,26 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 import * as winston from 'winston';
 
-export function createNewRootLogger(): winston.Logger {
-  return winston.createLogger({
-    level: process.env.LOG_LEVEL || 'info',
-    format:
-      process.env.NODE_ENV === 'production'
-        ? winston.format.json()
-        : winston.format.combine(
-            winston.format.colorize(),
-            winston.format.timestamp(),
-            winston.format.simple(),
-          ),
-    defaultMeta: { service: 'backstage' },
-    transports: [
-      new winston.transports.Console({
-        silent:
-          process.env.JEST_WORKER_ID !== undefined && !process.env.LOG_LEVEL,
-      }),
-    ],
-  });
-}
-
-let rootLogger: winston.Logger = createNewRootLogger();
+let rootLogger: winston.Logger = winston.createLogger({
+  level: process.env.LOG_LEVEL || 'info',
+  format:
+    process.env.NODE_ENV === 'production'
+      ? winston.format.json()
+      : winston.format.combine(
+          winston.format.colorize(),
+          winston.format.timestamp(),
+          winston.format.simple(),
+        ),
+  defaultMeta: { service: 'backstage' },
+  transports: [
+    new winston.transports.Console({
+      silent:
+        process.env.JEST_WORKER_ID !== undefined && !process.env.LOG_LEVEL,
+    }),
+  ],
+});
 
 export function getRootLogger(): winston.Logger {
   return rootLogger;

plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { PassThrough } from 'stream';
+import winston from 'winston';
+import { JsonValue } from '@backstage/config';
+
+export const useLogStream = (meta: Record<string, JsonValue>) => {
+  const log: string[] = [];
+
+  // Create an empty stream to collect all the log lines into
+  // one variable for the API.
+  const stream = new PassThrough();
+  stream.on('data', chunk => log.push(chunk.toString()));
+
+  const logger = winston.createLogger({
+    level: process.env.LOG_LEVEL || 'info',
+    format: winston.format.combine(
+      winston.format.colorize(),
+      winston.format.timestamp(),
+    ),
+    defaultMeta: meta,
+  });
+
+  logger.add(new winston.transports.Stream({ stream }));
+
+  return {
+    log,
+    stream,
+    logger,
+  };
+};

plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts

@@ -13,16 +13,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Processor, Job, Stage, StageContext } from './types';
+import { Processor, Job, Stage, StageContext, StageInput } from './types';
 import { JsonValue } from '@backstage/config';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
-import { PassThrough } from 'stream';
 import * as uuid from 'uuid';
 import Docker from 'dockerode';
-import winston from 'winston';
 import { RequiredTemplateValues, TemplaterBase } from '../stages/templater';
-import { createNewRootLogger } from '@backstage/backend-common';
 import { PreparerBuilder } from '../stages/prepare';
+import { useLogStream } from './logger';
 
 export type JobProcessorArguments = {
   preparers: PreparerBuilder;
@@ -37,29 +35,18 @@ export type JobAndDirectoryTuple = {
 
 export class JobProcessor implements Processor {
   private jobs = new Map<string, Job>();
-  private stages: Stage[];
-  constructor({ stages }: { stages: Stage[] }) {
-    this.stages = stages;
-  }
 
-  create(
-    entity: TemplateEntityV1alpha1,
-    values: RequiredTemplateValues & Record<string, JsonValue>,
-  ): Job {
+  create({
+    entity,
+    values,
+    stages,
+  }: {
+    entity: TemplateEntityV1alpha1;
+    values: RequiredTemplateValues & Record<string, JsonValue>;
+    stages: StageInput[];
+  }): Job {
     const id = uuid.v4();
-    const log: string[] = [];
-
-    // Create an empty stream to collect all the log lines into
-    // one variable for the API.
-    const logStream = new PassThrough();
-    logStream.on('data', chunk => log.push(chunk.toString()));
-
-    // TODO(blam): Maybe this is not the right way to build the logger
-    // Maybe we want to be more ux specific and drop the json support.
-    // Child loggers can not have specific transports which sucks, so we have to
-    // create another here.
-    const logger = createNewRootLogger();
-    logger.add(new winston.transports.Stream({ stream: logStream }));
+    const { logger, stream } = useLogStream({ id });
 
     const context: StageContext = {
       entity,
@@ -69,14 +56,10 @@ export class JobProcessor implements Processor {
 
     const job: Job = {
       id,
-      logStream,
+      logStream: stream,
       context,
-      stages: this.stages.map((stage) => ({}))
+      stages,
       status: 'PENDING',
-      metadata: {
-        entity,
-        values,
-      },
     };
 
     this.jobs.set(job.id, job);

plugins/scaffolder-backend/src/scaffolder/jobs/types.ts

@@ -22,45 +22,46 @@ import { Logger } from 'winston';
 // Context will be a mutable object which is passed between stages
 // To share data, but also thinking that we can pass in functions here too
 // To maybe create sub steps or fail the entire thing, or skip stages down the line.
-export type StageContext<T = {}> = T & {
+export type StageContext<T = {}> = {
   values: RequiredTemplateValues & Record<string, JsonValue>;
   entity: TemplateEntityV1alpha1;
   logger: Logger;
-};
+  logStream: Writable;
+} & T;
 
-export type Stage<T = {}> = {
+export type ProcessorStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED';
+
+export interface Stage extends StageInput {
   log: string[];
-  status: 'PENDING' | 'STARTED' | 'COMPLETE' | 'FAILED';
-  name: string;
-  handler: (ctx: StageContext<T>) => Promise<void>;
-};
+  status: ProcessorStatus;
+  startedAt?: number;
+  endedAt?: number;
+}
 
-export type Job<T = {}> = {
+export interface StageInput<T = {}> {
+  name: string;
+  handler(ctx: StageContext<T>): Promise<void | object>;
+}
+
+export type Job = {
   id: string;
-  context: StageContext<T>;
-  status: 'PENDING' | 'STARTED' | 'COMPLETE' | 'FAILED';
+  context: StageContext;
+  status: ProcessorStatus;
   stages: Stage[];
   logStream: Writable;
-  logger: Logger;
   error?: Error;
 };
 
-export interface ProcessorConstructor {
-  new (t: string): Processor;
-}
-
-export interface Processor {
-  create<T = {}>({
+export type Processor = {
+  create({
     entity,
     values,
     stages,
   }: {
     entity: TemplateEntityV1alpha1;
     values: RequiredTemplateValues & Record<string, JsonValue>;
-    stages: Stage[];
-  }): Job<T>;
+    stages: StageInput[];
+  }): Job;
 
   get(id: string): Job | undefined;
-}
-
-declare let Processor: ProcessorConstructor;
+};

plugins/scaffolder-backend/src/service/router.ts

@@ -21,6 +21,7 @@ import { PreparerBuilder, TemplaterBase, JobProcessor } from '../scaffolder';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import Docker from 'dockerode';
 import {} from '@backstage/backend-common';
+import { StageContext, Stage } from '../scaffolder/jobs/types';
 export interface RouterOptions {
   preparers: PreparerBuilder;
   templater: TemplaterBase;
@@ -35,11 +36,7 @@ export async function createRouter(
   const { preparers, templater, logger: parentLogger, dockerClient } = options;
   const logger = parentLogger.child({ plugin: 'scaffolder' });
 
-  const jobProcessor = new JobProcessor({
-    preparers,
-    templater,
-    dockerClient,
-  });
+  const jobProcessor = new JobProcessor();
 
   router
     .get('/v1/job/:jobId', ({ params }, res) => {
@@ -86,7 +83,45 @@ export async function createRouter(
         },
       };
 
-      const job = jobProcessor.create(mockEntity, { component_id: 'test' });
+      const job = jobProcessor.create({
+        entity: mockEntity,
+        values: { component_id: 'blob' },
+        stages: [
+          {
+            name: 'Prepare the skeleton',
+            handler: async ctx => {
+              const preparer = preparers.get(ctx.entity);
+              const skeletonDir = await preparer.prepare(ctx.entity);
+              return { skeletonDir };
+            },
+          },
+          {
+            name: 'Run the templater',
+            handler: async (ctx: StageContext<{ skeletonDir: string }>) => {
+              const resultDir = await templater.run({
+                directory: ctx.skeletonDir,
+                dockerClient,
+                logStream: ctx.logStream,
+                values: ctx.values,
+              });
+
+              return { resultDir };
+            },
+          },
+          {
+            name: 'Create VCS Repo',
+            handler: async (ctx: StageContext<{ resultDir: string }>) => {
+              ctx.logger.info('Should now create the VCS repo');
+            },
+          },
+          {
+            name: 'Push to remote',
+            handler: async ctx => {
+              ctx.logger.info('Should now push to the remote');
+            },
+          },
+        ],
+      });
       res.status(201).json({ jobId: job.id });
 
       jobProcessor.process(job);