diff --git a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml index b885357ff9..007e23a3d6 100644 --- a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml +++ b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml @@ -12,13 +12,17 @@ spec: properties: inputString: type: string + title: string input test inputObject: type: object + title: object input test properties: first: type: string + title: first second: type: number + title: second steps: - id: debug name: Debug diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts index e508d619c4..dea94e9b52 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -55,6 +55,14 @@ describe('DefaultWorkflowRunner', () => { handler: fakeActionHandler, }); + actionRegistry.register({ + id: 'output-action', + description: 'Mock action for testing', + handler: async ctx => { + ctx.output('mock', 'backstage'); + }, + }); + runner = new DefaultWorkflowRunner({ actionRegistry, integrations, @@ -189,5 +197,27 @@ describe('DefaultWorkflowRunner', () => { expect.objectContaining({ input: { foo: 1 } }), ); }); + + it('should template the output from simple actions', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + action: 'output-action', + input: {}, + }, + ], + output: { + foo: '${{steps.test.output.mock | upper}}', + }, + parameters: {}, + }); + + const { output } = await runner.execute(task); + + expect(output.foo).toEqual('BACKSTAGE'); + }); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index e84d7a71e6..c8f77f066a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -87,6 +87,43 @@ export class DefaultWorkflowRunner implements WorkflowRunner { }); } + private render(input: T, context: TemplateContext): T { + return JSON.parse(JSON.stringify(input), (_key, value) => { + try { + if (typeof value === 'string') { + const templated = this.nunjucks.renderString(value, context); + try { + return JSON.parse(templated); + } catch { + return templated; + } + } + } catch { + return value; + } + return value; + }); + } + + private makeStringifyableParams(input: T): T { + /** + * This is a little bit of a hack / magic so that when we use nunjucks and we try to + * pass through something other than a string from the parameters section. + * When an accessor is used that is an object, it's toString is the JSON.stringify'd version of it's children + * Which makes it work really well in string templating as we can parse the result again after.yarn + */ + return JSON.parse( + JSON.stringify(input), + (key: string, value: JsonObject) => { + if (typeof value === 'object' && key) { + value.toString = () => JSON.stringify(value); + } + + return value; + }, + ); + } + async execute(task: Task): Promise { if (!isValidTaskSpec(task.spec)) { throw new InputError( @@ -103,70 +140,61 @@ export class DefaultWorkflowRunner implements WorkflowRunner { `Starting up task with ${task.spec.steps.length} steps`, ); - /** - * This is a little bit of a hack / magic so that when we use nunjucks and we try to - * pass through an object from the `parameters` section of the task spec, it will - * actually work as the toString method is called from the nunjucks template. - */ - const parsedParams = JSON.parse( - JSON.stringify(task.spec.parameters), - (key: string, value: JsonObject) => { - if (typeof value === 'object' && key) { - value.toString = () => JSON.stringify(value); - } - - return value; - }, - ); - const context: TemplateContext = { - parameters: parsedParams, + parameters: this.makeStringifyableParams(task.spec.parameters), steps: {}, }; for (const step of task.spec.steps) { - const action = this.options.actionRegistry.get(step.action); - const { taskLogger, streamLogger } = createStepLogger({ task, step }); + try { + const action = this.options.actionRegistry.get(step.action); + const { taskLogger, streamLogger } = createStepLogger({ task, step }); - const input = - step.input && - JSON.parse(JSON.stringify(step.input), (_key, value) => { - try { - if (typeof value === 'string') { - const templated = this.nunjucks.renderString(value, context); - try { - return JSON.parse(templated); - } catch { - return templated; - } - } - } catch { - return value; - } - return value; + const input = step.input && this.render(step.input, context); + + const tmpDirs = new Array(); + const stepOutput: { [outputName: string]: JsonValue } = {}; + + await task.emitLog(`Beginning step ${step.name}`, { + stepId: step.id, + status: 'processing', }); - const tmpDirs = new Array(); - const stepOutputs: { [outputName: string]: JsonValue } = {}; + await action.handler({ + baseUrl: task.spec.baseUrl, + input, + logger: taskLogger, + logStream: streamLogger, + workspacePath, + createTemporaryDirectory: async () => { + const tmpDir = await fs.mkdtemp( + `${workspacePath}_step-${step.id}-`, + ); + tmpDirs.push(tmpDir); + return tmpDir; + }, + output(name: string, value: JsonValue) { + stepOutput[name] = value; + }, + }); - await action.handler({ - baseUrl: task.spec.baseUrl, - input, - logger: taskLogger, - logStream: streamLogger, - workspacePath, - createTemporaryDirectory: async () => { - const tmpDir = await fs.mkdtemp( - `${workspacePath}_step-${step.id}-`, - ); - tmpDirs.push(tmpDir); - return tmpDir; - }, - output(name: string, value: JsonValue) { - stepOutputs[name] = value; - }, - }); + context.steps[step.id] = { output: stepOutput }; + + await task.emitLog(`Finished step ${step.name}`, { + stepId: step.id, + status: 'completed', + }); + } catch (err) { + await task.emitLog(String(err.stack), { + stepId: step.id, + status: 'failed', + }); + } } + + const output = this.render(task.spec.output, context); + + return { output }; } finally { if (workspacePath) { await fs.remove(workspacePath); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts index a3c98f6cd4..97e3a4dec4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts @@ -18,6 +18,7 @@ import { WorkflowRunner, WorkflowResponse, TaskSpecV1beta2, + TaskSpec, } from './types'; import * as Handlebars from 'handlebars'; import { TemplateActionRegistry } from '..'; @@ -75,6 +76,7 @@ export class LegacyWorkflowRunner implements WorkflowRunner { if (!isValidTaskSpec(task.spec)) { throw new InputError(`Task spec is not a valid v1beta2 task spec`); } + const { actionRegistry } = this.options; const workspacePath = path.join( diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index 66c435862a..456836046c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -101,7 +101,7 @@ describe('TaskWorker', () => { output: { result: '{{ steps.test.output.testOutput }}', }, - values: {}, + parameters: {}, }); const task = await broker.claim(); @@ -130,7 +130,7 @@ describe('TaskWorker', () => { output: { result: '{{ steps.test.output.testOutput }}', }, - values: {}, + parameters: {}, }); const task = await broker.claim(); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 077d605083..b75a5d0368 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -42,6 +42,8 @@ import { import { ScmIntegrations } from '@backstage/integration'; import { TemplateAction } from '../scaffolder/actions'; import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; +import { LegacyWorkflowRunner } from '../scaffolder/tasks/LegacyWorkflowRunner'; +import { DefaultWorkflowRunner } from '../scaffolder/tasks/DefaultWorkflowRunner'; export interface RouterOptions { logger: Logger; @@ -90,14 +92,28 @@ export async function createRouter( ); const taskBroker = new StorageTaskBroker(databaseTaskStore, logger); const actionRegistry = new TemplateActionRegistry(); + const legacyWorkflowRunner = new LegacyWorkflowRunner({ + logger, + actionRegistry, + integrations, + workingDirectory, + }); + + const workflowRunner = new DefaultWorkflowRunner({ + actionRegistry, + integrations, + logger, + workingDirectory, + }); + const workers = []; for (let i = 0; i < (taskWorkers || 1); i++) { const worker = new TaskWorker({ - logger, taskBroker, - actionRegistry, - workingDirectory, - integrations, + runners: { + legacyWorkflowRunner, + workflowRunner, + }, }); workers.push(worker); } @@ -184,18 +200,32 @@ export async function createRouter( } const baseUrl = getEntityBaseUrl(template); - - taskSpec = { - apiVersion: template.apiVersion, - baseUrl, - values, - steps: template.spec.steps.map((step, index) => ({ - ...step, - id: step.id ?? `step-${index + 1}`, - name: step.name ?? step.action, - })), - output: template.spec.output ?? {}, - }; + // TODO: need to make sure that the TaskSpec is the right format here. + // If it's beta2 use values, beta3 uses parameters to clear that up. + taskSpec = + template.apiVersion === 'backstage.io/v1beta2' + ? { + apiVersion: template.apiVersion, + baseUrl, + values, + steps: template.spec.steps.map((step, index) => ({ + ...step, + id: step.id ?? `step-${index + 1}`, + name: step.name ?? step.action, + })), + output: template.spec.output ?? {}, + } + : { + apiVersion: template.apiVersion, + baseUrl, + parameters: values, + steps: template.spec.steps.map((step, index) => ({ + ...step, + id: step.id ?? `step-${index + 1}`, + name: step.name ?? step.action, + })), + output: template.spec.output ?? {}, + }; } else { throw new InputError( `Unsupported apiVersion field in schema entity, ${