feat(TaskWorker): move out the logic into the WorkflowRunner implementations

Signed-off-by: blam <ben@blam.sh>

packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts

@@ -30,6 +30,7 @@ describe('templateEntityV1beta3Validator', () => {
         name: 'test',
       },
       spec: {
+        type: 'website',
         parameters: {
           required: ['storePath', 'owner'],
           properties: {
@@ -38,11 +39,6 @@ describe('templateEntityV1beta3Validator', () => {
               title: 'Owner',
               description: 'Who is going to own this component',
             },
-            storePath: {
-              type: 'string',
-              title: 'Store path',
-              description: 'GitHub store path in org/repo format',
-            },
           },
         },
         steps: [

packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json

@@ -14,9 +14,8 @@
       },
       "spec": {
         "owner": "artist-relations-team",
-        "type": "website",
         "parameters": {
-          "required": ["name", "description"],
+          "required": ["name", "description", "repoUrl"],
           "properties": {
             "name": {
               "title": "Name",
@@ -27,6 +26,11 @@
               "title": "Description",
               "type": "string",
               "description": "Description of the component"
+            },
+            "repoUrl": {
+              "title": "Pick a repository",
+              "type": "string",
+              "ui:field": "RepoUrlPicker"
             }
           }
         },
@@ -44,13 +48,13 @@
             "name": "Publish to GitHub",
             "action": "publish:github",
             "parameters": {
-              "repoUrl": "{{ parameters.repoUrl }}"
+              "repoUrl": "${{ parameters.repoUrl }}"
             },
-            "if": "{{ parameters.repoUrl }}"
+            "if": "${{ parameters.repoUrl }}"
           }
         ],
         "output": {
-          "catalogInfoUrl": "{{ steps.publish.output.catalogInfoUrl }}"
+          "catalogInfoUrl": "${{ steps.publish.output.catalogInfoUrl }}"
         }
       }
     }

plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ScmIntegrations } from '@backstage/integration';
+import { TemplateActionRegistry } from '..';
+import { Task, WorkflowResponse, WorkflowRunner } from './types';
+
+type Options = {
+  workingDirectory: string;
+  actionRegistry: TemplateActionRegistry;
+  integrations: ScmIntegrations;
+};
+
+export class DefaultWorkflowRunner implements WorkflowRunner {
+  constructor(private readonly options: Options) {}
+  async execute(task: Task): Promise<WorkflowResponse> {
+    throw new Error('Method not implemented.');
+  }
+}

plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts

@@ -0,0 +1,297 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Task, WorkflowRunner, WorkflowResponse } from './types';
+import * as Handlebars from 'handlebars';
+import { TemplateActionRegistry } from '..';
+import { ScmIntegrations } from '@backstage/integration';
+import { parseRepoUrl } from '../actions/builtin/publish/util';
+import { isTruthy } from './helper';
+import { PassThrough } from 'stream';
+import * as winston from 'winston';
+import { Logger } from 'winston';
+import path from 'path';
+import fs from 'fs-extra';
+import { validate as validateJsonSchema } from 'jsonschema';
+import { JsonObject, JsonValue } from '@backstage/config';
+import { InputError } from '@backstage/errors';
+
+type Options = {
+  workingDirectory: string;
+  actionRegistry: TemplateActionRegistry;
+  integrations: ScmIntegrations;
+  logger: Logger;
+};
+
+/**
+ * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced
+ * with the default workflow runner interface in the future so this entire thing can go bye bye.
+ */
+export class LegacyWorkflowRunner implements WorkflowRunner {
+  private readonly handlebars: typeof Handlebars;
+
+  constructor(private readonly options: Options) {
+    this.handlebars = Handlebars.create();
+
+    // TODO(blam): this should be a public facing API but it's a little
+    // scary right now, so we're going to lock it off like the component API is
+    // in the frontend until we can work out a nice way to do it.
+    this.handlebars.registerHelper('parseRepoUrl', repoUrl => {
+      return JSON.stringify(parseRepoUrl(repoUrl, this.options.integrations));
+    });
+
+    this.handlebars.registerHelper('projectSlug', repoUrl => {
+      const { owner, repo } = parseRepoUrl(repoUrl, this.options.integrations);
+      return `${owner}/${repo}`;
+    });
+
+    this.handlebars.registerHelper('json', obj => JSON.stringify(obj));
+
+    this.handlebars.registerHelper('not', value => !isTruthy(value));
+
+    this.handlebars.registerHelper('eq', (a, b) => a === b);
+  }
+
+  async execute(task: Task): Promise<WorkflowResponse> {
+    const { actionRegistry } = this.options;
+
+    const workspacePath = path.join(
+      this.options.workingDirectory,
+      await task.getWorkspaceName(),
+    );
+    try {
+      await fs.ensureDir(workspacePath);
+      await task.emitLog(
+        `Starting up task with ${task.spec.steps.length} steps`,
+      );
+
+      const templateCtx: {
+        parameters: JsonObject;
+        steps: {
+          [stepName: string]: { output: { [outputName: string]: JsonValue } };
+        };
+      } = { parameters: task.spec.values, steps: {} };
+
+      for (const step of task.spec.steps) {
+        const metadata = { stepId: step.id };
+        try {
+          const taskLogger = winston.createLogger({
+            level: process.env.LOG_LEVEL || 'info',
+            format: winston.format.combine(
+              winston.format.colorize(),
+              winston.format.timestamp(),
+              winston.format.simple(),
+            ),
+            defaultMeta: {},
+          });
+
+          const stream = new PassThrough();
+          stream.on('data', async data => {
+            const message = data.toString().trim();
+            if (message?.length > 1) {
+              await task.emitLog(message, metadata);
+            }
+          });
+
+          taskLogger.add(new winston.transports.Stream({ stream }));
+
+          if (step.if !== undefined) {
+            // Support passing values like false to disable steps
+            let skip = !step.if;
+
+            // Evaluate strings as handlebar templates
+            if (typeof step.if === 'string') {
+              const condition = JSON.parse(
+                JSON.stringify(step.if),
+                (_key, value) => {
+                  if (typeof value === 'string') {
+                    const templated = this.handlebars.compile(value, {
+                      noEscape: true,
+                      data: false,
+                      preventIndent: true,
+                    })(templateCtx);
+
+                    // If it's just an empty string, treat it as undefined
+                    if (templated === '') {
+                      return undefined;
+                    }
+
+                    try {
+                      return JSON.parse(templated);
+                    } catch {
+                      return templated;
+                    }
+                  }
+
+                  return value;
+                },
+              );
+
+              skip = !isTruthy(condition);
+            }
+
+            if (skip) {
+              await task.emitLog(`Skipped step ${step.name}`, {
+                ...metadata,
+                status: 'skipped',
+              });
+              continue;
+            }
+          }
+
+          await task.emitLog(`Beginning step ${step.name}`, {
+            ...metadata,
+            status: 'processing',
+          });
+
+          const action = actionRegistry.get(step.action);
+          if (!action) {
+            throw new Error(`Action '${step.action}' does not exist`);
+          }
+
+          const input =
+            step.input &&
+            JSON.parse(JSON.stringify(step.input), (_key, value) => {
+              if (typeof value === 'string') {
+                const templated = this.handlebars.compile(value, {
+                  noEscape: true,
+                  data: false,
+                  preventIndent: true,
+                })(templateCtx);
+
+                // If it smells like a JSON object then give it a parse as an object and if it fails return the string
+                if (
+                  (templated.startsWith('"') && templated.endsWith('"')) ||
+                  (templated.startsWith('{') && templated.endsWith('}')) ||
+                  (templated.startsWith('[') && templated.endsWith(']'))
+                ) {
+                  try {
+                    // Don't recursively JSON parse the values of this string.
+                    // Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else
+                    return JSON.parse(templated);
+                  } catch {
+                    return templated;
+                  }
+                }
+                return templated;
+              }
+
+              return value;
+            });
+
+          if (action.schema?.input) {
+            const validateResult = validateJsonSchema(
+              input,
+              action.schema.input,
+            );
+            if (!validateResult.valid) {
+              const errors = validateResult.errors.join(', ');
+              throw new InputError(
+                `Invalid input passed to action ${action.id}, ${errors}`,
+              );
+            }
+          }
+
+          const stepOutputs: { [name: string]: JsonValue } = {};
+
+          // Keep track of all tmp dirs that are created by the action so we can remove them after
+          const tmpDirs = new Array<string>();
+
+          this.options.logger.debug(`Running ${action.id} with input`, {
+            input: JSON.stringify(input, null, 2),
+          });
+
+          await action.handler({
+            baseUrl: task.spec.baseUrl,
+            logger: taskLogger,
+            logStream: stream,
+            input,
+            token: task.secrets?.token,
+            workspacePath,
+            async createTemporaryDirectory() {
+              const tmpDir = await fs.mkdtemp(
+                `${workspacePath}_step-${step.id}-`,
+              );
+              tmpDirs.push(tmpDir);
+              return tmpDir;
+            },
+            output(name: string, value: JsonValue) {
+              stepOutputs[name] = value;
+            },
+          });
+
+          // Remove all temporary directories that were created when executing the action
+          for (const tmpDir of tmpDirs) {
+            await fs.remove(tmpDir);
+          }
+
+          templateCtx.steps[step.id] = { output: stepOutputs };
+
+          await task.emitLog(`Finished step ${step.name}`, {
+            ...metadata,
+            status: 'completed',
+          });
+        } catch (error) {
+          await task.emitLog(String(error.stack), {
+            ...metadata,
+            status: 'failed',
+          });
+          throw error;
+        }
+      }
+
+      const output = JSON.parse(
+        JSON.stringify(task.spec.output),
+        (_key, value) => {
+          if (typeof value === 'string') {
+            const templated = this.handlebars.compile(value, {
+              noEscape: true,
+              data: false,
+              preventIndent: true,
+            })(templateCtx);
+
+            // If it's just an empty string, treat it as undefined
+            if (templated === '') {
+              return undefined;
+            }
+
+            // If it smells like a JSON object then give it a parse as an object and if it fails return the string
+            if (
+              (templated.startsWith('"') && templated.endsWith('"')) ||
+              (templated.startsWith('{') && templated.endsWith('}')) ||
+              (templated.startsWith('[') && templated.endsWith(']'))
+            ) {
+              try {
+                // Don't recursively JSON parse the values of this string.
+                // Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else
+                return JSON.parse(templated);
+              } catch {
+                return templated;
+              }
+            }
+            return templated;
+          }
+          return value;
+        },
+      );
+
+      return { output };
+    } finally {
+      if (workspacePath) {
+        await fs.remove(workspacePath);
+      }
+    }
+  }
+}

plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts

@@ -17,17 +17,17 @@
 import { JsonObject, JsonValue } from '@backstage/config';
 import { InputError } from '@backstage/errors';
 import fs from 'fs-extra';
-import * as Handlebars from 'handlebars';
-import { validate as validateJsonSchema } from 'jsonschema';
+
 import path from 'path';
-import { PassThrough } from 'stream';
-import * as winston from 'winston';
+
 import { Logger } from 'winston';
 import { parseRepoUrl } from '../actions/builtin/publish/util';
 import { TemplateActionRegistry } from '../actions/TemplateActionRegistry';
 import { isTruthy } from './helper';
-import { Task, TaskBroker } from './types';
+import { Task, TaskBroker, WorkflowRunner } from './types';
 import { ScmIntegrations } from '@backstage/integration';
+import { LegacyWorkflowRunner } from './LegacyWorkflowRunner';
+import { DefaultWorkflowRunner } from './DefaultWorkflowRunner';
 
 type Options = {
   logger: Logger;
@@ -38,28 +38,12 @@ type Options = {
 };
 
 export class TaskWorker {
-  private readonly handlebars: typeof Handlebars;
+  private readonly legacyWorkflowRunner: LegacyWorkflowRunner;
+  private readonly workflowRunner: WorkflowRunner;
 
   constructor(private readonly options: Options) {
-    this.handlebars = Handlebars.create();
-
-    // TODO(blam): this should be a public facing API but it's a little
-    // scary right now, so we're going to lock it off like the component API is
-    // in the frontend until we can work out a nice way to do it.
-    this.handlebars.registerHelper('parseRepoUrl', repoUrl => {
-      return JSON.stringify(parseRepoUrl(repoUrl, options.integrations));
-    });
-
-    this.handlebars.registerHelper('projectSlug', repoUrl => {
-      const { owner, repo } = parseRepoUrl(repoUrl, options.integrations);
-      return `${owner}/${repo}`;
-    });
-
-    this.handlebars.registerHelper('json', obj => JSON.stringify(obj));
-
-    this.handlebars.registerHelper('not', value => !isTruthy(value));
-
-    this.handlebars.registerHelper('eq', (a, b) => a === b);
+    this.legacyWorkflowRunner = new LegacyWorkflowRunner(options);
+    this.workflowRunner = new DefaultWorkflowRunner(options);
   }
 
   start() {
@@ -72,247 +56,17 @@ export class TaskWorker {
   }
 
   async runOneTask(task: Task) {
-    let workspacePath: string | undefined = undefined;
     try {
-      const { actionRegistry } = this.options;
-
-      workspacePath = path.join(
-        this.options.workingDirectory,
-        await task.getWorkspaceName(),
-      );
-      await fs.ensureDir(workspacePath);
-      await task.emitLog(
-        `Starting up task with ${task.spec.steps.length} steps`,
-      );
-
-      const templateCtx: {
-        parameters: JsonObject;
-        steps: {
-          [stepName: string]: { output: { [outputName: string]: JsonValue } };
-        };
-      } = { parameters: task.spec.values, steps: {} };
-
       const { output } =
         task.spec.apiVersion === 'backstage.io/v1beta3'
-          ? await TemplateWorkflowRunner.execute(task)
-          : await LegacyWorkflowRunner.execute(task);
-      if (task.spec.apiVersion === 'backstage.io/v1beta3') {
-        const { output } = await TemplateWorkflowRunnger.execute(task);
-      } else {
-      }
-
-      for (const step of task.spec.steps) {
-        const metadata = { stepId: step.id };
-        try {
-          const taskLogger = winston.createLogger({
-            level: process.env.LOG_LEVEL || 'info',
-            format: winston.format.combine(
-              winston.format.colorize(),
-              winston.format.timestamp(),
-              winston.format.simple(),
-            ),
-            defaultMeta: {},
-          });
-
-          const stream = new PassThrough();
-          stream.on('data', async data => {
-            const message = data.toString().trim();
-            if (message?.length > 1) {
-              await task.emitLog(message, metadata);
-            }
-          });
-
-          taskLogger.add(new winston.transports.Stream({ stream }));
-
-          if (step.if !== undefined) {
-            // Support passing values like false to disable steps
-            let skip = !step.if;
-
-            // Evaluate strings as handlebar templates
-            if (typeof step.if === 'string') {
-              const condition = JSON.parse(
-                JSON.stringify(step.if),
-                (_key, value) => {
-                  if (typeof value === 'string') {
-                    const templated = this.handlebars.compile(value, {
-                      noEscape: true,
-                      data: false,
-                      preventIndent: true,
-                    })(templateCtx);
-
-                    // If it's just an empty string, treat it as undefined
-                    if (templated === '') {
-                      return undefined;
-                    }
-
-                    try {
-                      return JSON.parse(templated);
-                    } catch {
-                      return templated;
-                    }
-                  }
-
-                  return value;
-                },
-              );
-
-              skip = !isTruthy(condition);
-            }
-
-            if (skip) {
-              await task.emitLog(`Skipped step ${step.name}`, {
-                ...metadata,
-                status: 'skipped',
-              });
-              continue;
-            }
-          }
-
-          await task.emitLog(`Beginning step ${step.name}`, {
-            ...metadata,
-            status: 'processing',
-          });
-
-          const action = actionRegistry.get(step.action);
-          if (!action) {
-            throw new Error(`Action '${step.action}' does not exist`);
-          }
-
-          const input =
-            step.input &&
-            JSON.parse(JSON.stringify(step.input), (_key, value) => {
-              if (typeof value === 'string') {
-                const templated = this.handlebars.compile(value, {
-                  noEscape: true,
-                  data: false,
-                  preventIndent: true,
-                })(templateCtx);
-
-                // If it smells like a JSON object then give it a parse as an object and if it fails return the string
-                if (
-                  (templated.startsWith('"') && templated.endsWith('"')) ||
-                  (templated.startsWith('{') && templated.endsWith('}')) ||
-                  (templated.startsWith('[') && templated.endsWith(']'))
-                ) {
-                  try {
-                    // Don't recursively JSON parse the values of this string.
-                    // Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else
-                    return JSON.parse(templated);
-                  } catch {
-                    return templated;
-                  }
-                }
-                return templated;
-              }
-
-              return value;
-            });
-
-          if (action.schema?.input) {
-            const validateResult = validateJsonSchema(
-              input,
-              action.schema.input,
-            );
-            if (!validateResult.valid) {
-              const errors = validateResult.errors.join(', ');
-              throw new InputError(
-                `Invalid input passed to action ${action.id}, ${errors}`,
-              );
-            }
-          }
-
-          const stepOutputs: { [name: string]: JsonValue } = {};
-
-          // Keep track of all tmp dirs that are created by the action so we can remove them after
-          const tmpDirs = new Array<string>();
-
-          this.options.logger.debug(`Running ${action.id} with input`, {
-            input: JSON.stringify(input, null, 2),
-          });
-
-          await action.handler({
-            baseUrl: task.spec.baseUrl,
-            logger: taskLogger,
-            logStream: stream,
-            input,
-            token: task.secrets?.token,
-            workspacePath,
-            async createTemporaryDirectory() {
-              const tmpDir = await fs.mkdtemp(
-                `${workspacePath}_step-${step.id}-`,
-              );
-              tmpDirs.push(tmpDir);
-              return tmpDir;
-            },
-            output(name: string, value: JsonValue) {
-              stepOutputs[name] = value;
-            },
-          });
-
-          // Remove all temporary directories that were created when executing the action
-          for (const tmpDir of tmpDirs) {
-            await fs.remove(tmpDir);
-          }
-
-          templateCtx.steps[step.id] = { output: stepOutputs };
-
-          await task.emitLog(`Finished step ${step.name}`, {
-            ...metadata,
-            status: 'completed',
-          });
-        } catch (error) {
-          await task.emitLog(String(error.stack), {
-            ...metadata,
-            status: 'failed',
-          });
-          throw error;
-        }
-      }
-
-      const output = JSON.parse(
-        JSON.stringify(task.spec.output),
-        (_key, value) => {
-          if (typeof value === 'string') {
-            const templated = this.handlebars.compile(value, {
-              noEscape: true,
-              data: false,
-              preventIndent: true,
-            })(templateCtx);
-
-            // If it's just an empty string, treat it as undefined
-            if (templated === '') {
-              return undefined;
-            }
-
-            // If it smells like a JSON object then give it a parse as an object and if it fails return the string
-            if (
-              (templated.startsWith('"') && templated.endsWith('"')) ||
-              (templated.startsWith('{') && templated.endsWith('}')) ||
-              (templated.startsWith('[') && templated.endsWith(']'))
-            ) {
-              try {
-                // Don't recursively JSON parse the values of this string.
-                // Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else
-                return JSON.parse(templated);
-              } catch {
-                return templated;
-              }
-            }
-            return templated;
-          }
-          return value;
-        },
-      );
+          ? await this.workflowRunner.execute(task)
+          : await this.legacyWorkflowRunner.execute(task);
 
       await task.complete('completed', { output });
     } catch (error) {
       await task.complete('failed', {
         error: { name: error.name, message: error.message },
       });
-    } finally {
-      if (workspacePath) {
-        await fs.remove(workspacePath);
-      }
     }
   }
 }

plugins/scaffolder-backend/src/scaffolder/tasks/types.ts

@@ -44,6 +44,7 @@ export type DbTaskEventRow = {
 };
 
 export type TaskSpec = {
+  apiVersion: 'backstage.io/v1beta2' | 'backstage.io/v1beta3';
   baseUrl?: string;
   values: JsonObject;
   steps: Array<{
@@ -122,3 +123,8 @@ export interface TaskStore {
     after,
   }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }>;
 }
+
+export type WorkflowResponse = { output: { [name: string]: JsonValue } };
+export interface WorkflowRunner {
+  execute(task: Task): Promise<WorkflowResponse>;
+}