diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts new file mode 100644 index 0000000000..e508d619c4 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -0,0 +1,193 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import os from 'os'; +import { getVoidLogger } from '@backstage/backend-common'; +import { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; +import { TemplateActionRegistry } from '../actions'; +import { ScmIntegrations } from '@backstage/integration'; +import { ConfigReader } from '@backstage/config'; +import { Task, TaskSpec } from './types'; + +describe('DefaultWorkflowRunner', () => { + const workingDirectory = os.tmpdir(); + const logger = getVoidLogger(); + let actionRegistry = new TemplateActionRegistry(); + let runner: DefaultWorkflowRunner; + let fakeActionHandler: jest.Mock; + + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'token' }], + }, + }), + ); + + const createMockTaskWithSpec = (spec: TaskSpec): Task => ({ + spec, + complete: async () => {}, + done: false, + emitLog: async () => {}, + getWorkspaceName: () => Promise.resolve('test-workspace'), + }); + + beforeEach(() => { + jest.resetAllMocks(); + actionRegistry = new TemplateActionRegistry(); + fakeActionHandler = jest.fn(); + + actionRegistry.register({ + id: 'jest-mock-action', + description: 'Mock action for testing', + handler: fakeActionHandler, + }); + + runner = new DefaultWorkflowRunner({ + actionRegistry, + integrations, + workingDirectory, + logger, + }); + }); + + it('should throw an error if the action does not exist', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + parameters: {}, + output: {}, + steps: [{ id: 'test', name: 'name', action: 'does-not-exist' }], + }); + + await expect(runner.execute(task)).rejects.toThrowError( + "Template action with ID 'does-not-exist' is not registered.", + ); + }); + + describe('validation', () => {}); + describe('running', () => {}); + + describe('templating', () => { + it('should template the input to an action', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'jest-mock-action', + input: { + foo: '${{parameters.input | lower }}', + }, + }, + ], + output: {}, + parameters: { + input: 'BACKSTAGE', + }, + }); + + await runner.execute(task); + + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ input: { foo: 'backstage' } }), + ); + }); + + it('should template complex values into the action', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'jest-mock-action', + input: { + foo: '${{parameters.complex}}', + }, + }, + ], + output: {}, + parameters: { + complex: { bar: 'BACKSTAGE' }, + }, + }); + + await runner.execute(task); + + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ input: { foo: { bar: 'BACKSTAGE' } } }), + ); + }); + + it('supports really complex structures', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'jest-mock-action', + input: { + foo: '${{parameters.complex.baz.something}}', + }, + }, + ], + output: {}, + parameters: { + complex: { + bar: 'BACKSTAGE', + baz: { something: 'nested', here: 'yas' }, + }, + }, + }); + + await runner.execute(task); + + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ input: { foo: 'nested' } }), + ); + }); + + it('supports numbers as first class too', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'jest-mock-action', + input: { + foo: '${{parameters.complex.baz.number}}', + }, + }, + ], + output: {}, + parameters: { + complex: { + bar: 'BACKSTAGE', + baz: { number: 1 }, + }, + }, + }); + + await runner.execute(task); + + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ input: { foo: 1 } }), + ); + }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index 813fa1b363..e84d7a71e6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -15,17 +15,162 @@ */ import { ScmIntegrations } from '@backstage/integration'; import { TemplateActionRegistry } from '..'; -import { Task, WorkflowResponse, WorkflowRunner } from './types'; +import { + Task, + TaskSpec, + TaskSpecV1beta3, + TaskStep, + WorkflowResponse, + WorkflowRunner, +} from './types'; +import * as winston from 'winston'; +import nunjucks from 'nunjucks'; +import fs from 'fs-extra'; +import path from 'path'; +import { JsonObject, JsonValue } from '@backstage/config'; +import { InputError } from '@backstage/errors'; +import { PassThrough } from 'stream'; type Options = { workingDirectory: string; actionRegistry: TemplateActionRegistry; integrations: ScmIntegrations; + logger: winston.Logger; +}; + +type TemplateContext = { + parameters: JsonObject; + steps: { + [stepName: string]: { output: { [outputName: string]: JsonValue } }; + }; +}; + +const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta3 => { + return taskSpec.apiVersion === 'backstage.io/v1beta3'; +}; + +const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { + const metadata = { stepId: step.id }; + const taskLogger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.colorize(), + winston.format.timestamp(), + winston.format.simple(), + ), + defaultMeta: {}, + }); + + const streamLogger = new PassThrough(); + streamLogger.on('data', async data => { + const message = data.toString().trim(); + if (message?.length > 1) { + await task.emitLog(message, metadata); + } + }); + + taskLogger.add(new winston.transports.Stream({ stream: streamLogger })); + + return { taskLogger, streamLogger }; }; export class DefaultWorkflowRunner implements WorkflowRunner { - constructor(private readonly options: Options) {} + private readonly nunjucks: nunjucks.Environment; + + constructor(private readonly options: Options) { + this.nunjucks = nunjucks.configure({ + autoescape: false, + tags: { + variableStart: '${{', + variableEnd: '}}', + }, + }); + } + async execute(task: Task): Promise { - throw new Error('Method not implemented.'); + if (!isValidTaskSpec(task.spec)) { + throw new InputError( + 'Wrong template version executed with the workflow engine', + ); + } + const workspacePath = path.join( + this.options.workingDirectory, + await task.getWorkspaceName(), + ); + try { + await fs.ensureDir(workspacePath); + await task.emitLog( + `Starting up task with ${task.spec.steps.length} steps`, + ); + + /** + * This is a little bit of a hack / magic so that when we use nunjucks and we try to + * pass through an object from the `parameters` section of the task spec, it will + * actually work as the toString method is called from the nunjucks template. + */ + const parsedParams = JSON.parse( + JSON.stringify(task.spec.parameters), + (key: string, value: JsonObject) => { + if (typeof value === 'object' && key) { + value.toString = () => JSON.stringify(value); + } + + return value; + }, + ); + + const context: TemplateContext = { + parameters: parsedParams, + steps: {}, + }; + + for (const step of task.spec.steps) { + const action = this.options.actionRegistry.get(step.action); + const { taskLogger, streamLogger } = createStepLogger({ task, step }); + + const input = + step.input && + JSON.parse(JSON.stringify(step.input), (_key, value) => { + try { + if (typeof value === 'string') { + const templated = this.nunjucks.renderString(value, context); + try { + return JSON.parse(templated); + } catch { + return templated; + } + } + } catch { + return value; + } + return value; + }); + + const tmpDirs = new Array(); + const stepOutputs: { [outputName: string]: JsonValue } = {}; + + await action.handler({ + baseUrl: task.spec.baseUrl, + input, + logger: taskLogger, + logStream: streamLogger, + workspacePath, + createTemporaryDirectory: async () => { + const tmpDir = await fs.mkdtemp( + `${workspacePath}_step-${step.id}-`, + ); + tmpDirs.push(tmpDir); + return tmpDir; + }, + output(name: string, value: JsonValue) { + stepOutputs[name] = value; + }, + }); + } + } finally { + if (workspacePath) { + await fs.remove(workspacePath); + } + } } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts index a5bf740c07..a3c98f6cd4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts @@ -13,7 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Task, WorkflowRunner, WorkflowResponse } from './types'; +import { + Task, + WorkflowRunner, + WorkflowResponse, + TaskSpecV1beta2, +} from './types'; import * as Handlebars from 'handlebars'; import { TemplateActionRegistry } from '..'; import { ScmIntegrations } from '@backstage/integration'; @@ -35,6 +40,8 @@ type Options = { logger: Logger; }; +const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta2 => + taskSpec.apiVersion === 'backstage.io/v1beta2'; /** * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced * with the default workflow runner interface in the future so this entire thing can go bye bye. @@ -65,6 +72,9 @@ export class LegacyWorkflowRunner implements WorkflowRunner { } async execute(task: Task): Promise { + if (!isValidTaskSpec(task.spec)) { + throw new InputError(`Task spec is not a valid v1beta2 task spec`); + } const { actionRegistry } = this.options; const workspacePath = path.join( diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 5ff40b9459..2ab38edec9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -43,8 +43,8 @@ export type DbTaskEventRow = { createdAt: string; }; -export type TaskSpec = { - apiVersion: 'backstage.io/v1beta2' | 'backstage.io/v1beta3'; +export interface TaskSpecV1beta2 { + apiVersion: 'backstage.io/v1beta2'; baseUrl?: string; values: JsonObject; steps: Array<{ @@ -55,7 +55,24 @@ export type TaskSpec = { if?: string | boolean; }>; output: { [name: string]: string }; -}; +} + +export interface TaskStep { + id: string; + name: string; + action: string; + input?: JsonObject; + if?: string | boolean; +} +export interface TaskSpecV1beta3 { + apiVersion: 'backstage.io/v1beta3'; + baseUrl?: string; + parameters: JsonObject; + steps: TaskStep[]; + output: { [name: string]: string }; +} + +export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3; export type TaskSecrets = { token: string | undefined;