chore: worked out a niceish way to template objects and stuff as first class citizens

Signed-off-by: blam <ben@blam.sh>

plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts

@@ -0,0 +1,193 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import os from 'os';
+import { getVoidLogger } from '@backstage/backend-common';
+import { DefaultWorkflowRunner } from './DefaultWorkflowRunner';
+import { TemplateActionRegistry } from '../actions';
+import { ScmIntegrations } from '@backstage/integration';
+import { ConfigReader } from '@backstage/config';
+import { Task, TaskSpec } from './types';
+
+describe('DefaultWorkflowRunner', () => {
+  const workingDirectory = os.tmpdir();
+  const logger = getVoidLogger();
+  let actionRegistry = new TemplateActionRegistry();
+  let runner: DefaultWorkflowRunner;
+  let fakeActionHandler: jest.Mock;
+
+  const integrations = ScmIntegrations.fromConfig(
+    new ConfigReader({
+      integrations: {
+        github: [{ host: 'github.com', token: 'token' }],
+      },
+    }),
+  );
+
+  const createMockTaskWithSpec = (spec: TaskSpec): Task => ({
+    spec,
+    complete: async () => {},
+    done: false,
+    emitLog: async () => {},
+    getWorkspaceName: () => Promise.resolve('test-workspace'),
+  });
+
+  beforeEach(() => {
+    jest.resetAllMocks();
+    actionRegistry = new TemplateActionRegistry();
+    fakeActionHandler = jest.fn();
+
+    actionRegistry.register({
+      id: 'jest-mock-action',
+      description: 'Mock action for testing',
+      handler: fakeActionHandler,
+    });
+
+    runner = new DefaultWorkflowRunner({
+      actionRegistry,
+      integrations,
+      workingDirectory,
+      logger,
+    });
+  });
+
+  it('should throw an error if the action does not exist', async () => {
+    const task = createMockTaskWithSpec({
+      apiVersion: 'backstage.io/v1beta3',
+      parameters: {},
+      output: {},
+      steps: [{ id: 'test', name: 'name', action: 'does-not-exist' }],
+    });
+
+    await expect(runner.execute(task)).rejects.toThrowError(
+      "Template action with ID 'does-not-exist' is not registered.",
+    );
+  });
+
+  describe('validation', () => {});
+  describe('running', () => {});
+
+  describe('templating', () => {
+    it('should template the input to an action', async () => {
+      const task = createMockTaskWithSpec({
+        apiVersion: 'backstage.io/v1beta3',
+        steps: [
+          {
+            id: 'test',
+            name: 'name',
+            action: 'jest-mock-action',
+            input: {
+              foo: '${{parameters.input | lower }}',
+            },
+          },
+        ],
+        output: {},
+        parameters: {
+          input: 'BACKSTAGE',
+        },
+      });
+
+      await runner.execute(task);
+
+      expect(fakeActionHandler).toHaveBeenCalledWith(
+        expect.objectContaining({ input: { foo: 'backstage' } }),
+      );
+    });
+
+    it('should template complex values into the action', async () => {
+      const task = createMockTaskWithSpec({
+        apiVersion: 'backstage.io/v1beta3',
+        steps: [
+          {
+            id: 'test',
+            name: 'name',
+            action: 'jest-mock-action',
+            input: {
+              foo: '${{parameters.complex}}',
+            },
+          },
+        ],
+        output: {},
+        parameters: {
+          complex: { bar: 'BACKSTAGE' },
+        },
+      });
+
+      await runner.execute(task);
+
+      expect(fakeActionHandler).toHaveBeenCalledWith(
+        expect.objectContaining({ input: { foo: { bar: 'BACKSTAGE' } } }),
+      );
+    });
+
+    it('supports really complex structures', async () => {
+      const task = createMockTaskWithSpec({
+        apiVersion: 'backstage.io/v1beta3',
+        steps: [
+          {
+            id: 'test',
+            name: 'name',
+            action: 'jest-mock-action',
+            input: {
+              foo: '${{parameters.complex.baz.something}}',
+            },
+          },
+        ],
+        output: {},
+        parameters: {
+          complex: {
+            bar: 'BACKSTAGE',
+            baz: { something: 'nested', here: 'yas' },
+          },
+        },
+      });
+
+      await runner.execute(task);
+
+      expect(fakeActionHandler).toHaveBeenCalledWith(
+        expect.objectContaining({ input: { foo: 'nested' } }),
+      );
+    });
+
+    it('supports numbers as first class too', async () => {
+      const task = createMockTaskWithSpec({
+        apiVersion: 'backstage.io/v1beta3',
+        steps: [
+          {
+            id: 'test',
+            name: 'name',
+            action: 'jest-mock-action',
+            input: {
+              foo: '${{parameters.complex.baz.number}}',
+            },
+          },
+        ],
+        output: {},
+        parameters: {
+          complex: {
+            bar: 'BACKSTAGE',
+            baz: { number: 1 },
+          },
+        },
+      });
+
+      await runner.execute(task);
+
+      expect(fakeActionHandler).toHaveBeenCalledWith(
+        expect.objectContaining({ input: { foo: 1 } }),
+      );
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts

@@ -15,17 +15,162 @@
  */
 import { ScmIntegrations } from '@backstage/integration';
 import { TemplateActionRegistry } from '..';
-import { Task, WorkflowResponse, WorkflowRunner } from './types';
+import {
+  Task,
+  TaskSpec,
+  TaskSpecV1beta3,
+  TaskStep,
+  WorkflowResponse,
+  WorkflowRunner,
+} from './types';
+import * as winston from 'winston';
+import nunjucks from 'nunjucks';
+import fs from 'fs-extra';
+import path from 'path';
+import { JsonObject, JsonValue } from '@backstage/config';
+import { InputError } from '@backstage/errors';
+import { PassThrough } from 'stream';
 
 type Options = {
   workingDirectory: string;
   actionRegistry: TemplateActionRegistry;
   integrations: ScmIntegrations;
+  logger: winston.Logger;
+};
+
+type TemplateContext = {
+  parameters: JsonObject;
+  steps: {
+    [stepName: string]: { output: { [outputName: string]: JsonValue } };
+  };
+};
+
+const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta3 => {
+  return taskSpec.apiVersion === 'backstage.io/v1beta3';
+};
+
+const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => {
+  const metadata = { stepId: step.id };
+  const taskLogger = winston.createLogger({
+    level: process.env.LOG_LEVEL || 'info',
+    format: winston.format.combine(
+      winston.format.colorize(),
+      winston.format.timestamp(),
+      winston.format.simple(),
+    ),
+    defaultMeta: {},
+  });
+
+  const streamLogger = new PassThrough();
+  streamLogger.on('data', async data => {
+    const message = data.toString().trim();
+    if (message?.length > 1) {
+      await task.emitLog(message, metadata);
+    }
+  });
+
+  taskLogger.add(new winston.transports.Stream({ stream: streamLogger }));
+
+  return { taskLogger, streamLogger };
 };
 
 export class DefaultWorkflowRunner implements WorkflowRunner {
-  constructor(private readonly options: Options) {}
+  private readonly nunjucks: nunjucks.Environment;
+
+  constructor(private readonly options: Options) {
+    this.nunjucks = nunjucks.configure({
+      autoescape: false,
+      tags: {
+        variableStart: '${{',
+        variableEnd: '}}',
+      },
+    });
+  }
+
   async execute(task: Task): Promise<WorkflowResponse> {
-    throw new Error('Method not implemented.');
+    if (!isValidTaskSpec(task.spec)) {
+      throw new InputError(
+        'Wrong template version executed with the workflow engine',
+      );
+    }
+    const workspacePath = path.join(
+      this.options.workingDirectory,
+      await task.getWorkspaceName(),
+    );
+    try {
+      await fs.ensureDir(workspacePath);
+      await task.emitLog(
+        `Starting up task with ${task.spec.steps.length} steps`,
+      );
+
+      /**
+       * This is a little bit of a hack / magic so that when we use nunjucks and we try to
+       * pass through an object from the `parameters` section of the task spec, it will
+       * actually work as the toString method is called from the nunjucks template.
+       */
+      const parsedParams = JSON.parse(
+        JSON.stringify(task.spec.parameters),
+        (key: string, value: JsonObject) => {
+          if (typeof value === 'object' && key) {
+            value.toString = () => JSON.stringify(value);
+          }
+
+          return value;
+        },
+      );
+
+      const context: TemplateContext = {
+        parameters: parsedParams,
+        steps: {},
+      };
+
+      for (const step of task.spec.steps) {
+        const action = this.options.actionRegistry.get(step.action);
+        const { taskLogger, streamLogger } = createStepLogger({ task, step });
+
+        const input =
+          step.input &&
+          JSON.parse(JSON.stringify(step.input), (_key, value) => {
+            try {
+              if (typeof value === 'string') {
+                const templated = this.nunjucks.renderString(value, context);
+                try {
+                  return JSON.parse(templated);
+                } catch {
+                  return templated;
+                }
+              }
+            } catch {
+              return value;
+            }
+            return value;
+          });
+
+        const tmpDirs = new Array<string>();
+        const stepOutputs: { [outputName: string]: JsonValue } = {};
+
+        await action.handler({
+          baseUrl: task.spec.baseUrl,
+          input,
+          logger: taskLogger,
+          logStream: streamLogger,
+          workspacePath,
+          createTemporaryDirectory: async () => {
+            const tmpDir = await fs.mkdtemp(
+              `${workspacePath}_step-${step.id}-`,
+            );
+            tmpDirs.push(tmpDir);
+            return tmpDir;
+          },
+          output(name: string, value: JsonValue) {
+            stepOutputs[name] = value;
+          },
+        });
+      }
+    } finally {
+      if (workspacePath) {
+        await fs.remove(workspacePath);
+      }
+    }
   }
 }

plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts

@@ -13,7 +13,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Task, WorkflowRunner, WorkflowResponse } from './types';
+import {
+  Task,
+  WorkflowRunner,
+  WorkflowResponse,
+  TaskSpecV1beta2,
+} from './types';
 import * as Handlebars from 'handlebars';
 import { TemplateActionRegistry } from '..';
 import { ScmIntegrations } from '@backstage/integration';
@@ -35,6 +40,8 @@ type Options = {
   logger: Logger;
 };
 
+const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta2 =>
+  taskSpec.apiVersion === 'backstage.io/v1beta2';
 /**
  * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced
  * with the default workflow runner interface in the future so this entire thing can go bye bye.
@@ -65,6 +72,9 @@ export class LegacyWorkflowRunner implements WorkflowRunner {
   }
 
   async execute(task: Task): Promise<WorkflowResponse> {
+    if (!isValidTaskSpec(task.spec)) {
+      throw new InputError(`Task spec is not a valid v1beta2 task spec`);
+    }
     const { actionRegistry } = this.options;
 
     const workspacePath = path.join(

plugins/scaffolder-backend/src/scaffolder/tasks/types.ts

@@ -43,8 +43,8 @@ export type DbTaskEventRow = {
   createdAt: string;
 };
 
-export type TaskSpec = {
-  apiVersion: 'backstage.io/v1beta2' | 'backstage.io/v1beta3';
+export interface TaskSpecV1beta2 {
+  apiVersion: 'backstage.io/v1beta2';
   baseUrl?: string;
   values: JsonObject;
   steps: Array<{
@@ -55,7 +55,24 @@ export type TaskSpec = {
     if?: string | boolean;
   }>;
   output: { [name: string]: string };
-};
+}
+
+export interface TaskStep {
+  id: string;
+  name: string;
+  action: string;
+  input?: JsonObject;
+  if?: string | boolean;
+}
+export interface TaskSpecV1beta3 {
+  apiVersion: 'backstage.io/v1beta3';
+  baseUrl?: string;
+  parameters: JsonObject;
+  steps: TaskStep[];
+  output: { [name: string]: string };
+}
+
+export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3;
 
 export type TaskSecrets = {
   token: string | undefined;