From a0303ee5ef95977dca3c8dbbe98b287af82c1ddb Mon Sep 17 00:00:00 2001 From: Alex Eftimie Date: Wed, 7 Jun 2023 03:46:41 +0200 Subject: [PATCH 1/8] scaffolder: enable each property on task step (#8890) Signed-off-by: Alex Eftimie --- .../tasks/NunjucksWorkflowRunner.test.ts | 33 +++++++++ .../tasks/NunjucksWorkflowRunner.ts | 70 ++++++++++++++----- plugins/scaffolder-common/src/TaskSpec.ts | 6 +- plugins/scaffolder-node/src/actions/types.ts | 5 ++ 4 files changed, 94 insertions(+), 20 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts index 5d4f7b73cc..8fbbd45b94 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts @@ -573,6 +573,39 @@ describe('DefaultWorkflowRunner', () => { }); }); + describe('each', () => { + it('should run a step repeatedly', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'scaffolder.backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + each: '${{parameters.colors}}', + action: 'jest-mock-action', + input: { color: '${{each}}' }, + }, + ], + output: {}, + parameters: { + colors: ['blue', 'green', 'red'], + }, + }); + + await runner.execute(task); + + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ input: { color: 'blue' } }), + ); + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ input: { color: 'green' } }), + ); + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ input: { color: 'red' } }), + ); + }); + }); + describe('secrets', () => { it('should pass through the secrets to the context', async () => { const task = createMockTaskWithSpec( diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index da690bcb82..e3233483ef 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -76,6 +76,7 @@ type TemplateContext = { entity?: UserEntity; ref?: string; }; + each?: JsonValue; }; const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta3 => { @@ -311,25 +312,56 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { const tmpDirs = new Array(); const stepOutput: { [outputName: string]: JsonValue } = {}; - await action.handler({ - input, - secrets: task.secrets ?? {}, - logger: taskLogger, - logStream: streamLogger, - workspacePath, - createTemporaryDirectory: async () => { - const tmpDir = await fs.mkdtemp(`${workspacePath}_step-${step.id}-`); - tmpDirs.push(tmpDir); - return tmpDir; - }, - output(name: string, value: JsonValue) { - stepOutput[name] = value; - }, - templateInfo: task.spec.templateInfo, - user: task.spec.user, - isDryRun: task.isDryRun, - signal: task.cancelSignal, - }); + const iterations = new Array(); + if (step.each) { + const each = await this.render(step.each, context, renderTemplate); + iterations.push(...each); + } else { + iterations.push({}); + } + + let actionInput = input; + for (const iteration of iterations) { + if (step.each) { + taskLogger.info(`Running step each: ${iteration}`); + context.each = iteration; + // re-render input with the modified context that includes each + actionInput = + (step.input && + this.render( + step.input, + { ...context, secrets: task.secrets ?? {} }, + renderTemplate, + )) ?? + {}; + } + await action.handler({ + input: actionInput, + secrets: task.secrets ?? {}, + logger: taskLogger, + logStream: streamLogger, + workspacePath, + createTemporaryDirectory: async () => { + const tmpDir = await fs.mkdtemp( + `${workspacePath}_step-${step.id}-`, + ); + tmpDirs.push(tmpDir); + return tmpDir; + }, + output(name: string, value: JsonValue) { + if (step.each) { + stepOutput[name] = stepOutput[name] || []; + stepOutput[name].push(value); + } else { + stepOutput[name] = value; + } + }, + templateInfo: task.spec.templateInfo, + user: task.spec.user, + isDryRun: task.isDryRun, + signal: task.cancelSignal, + }); + } // Remove all temporary directories that were created when executing the action for (const tmpDir of tmpDirs) { diff --git a/plugins/scaffolder-common/src/TaskSpec.ts b/plugins/scaffolder-common/src/TaskSpec.ts index 7c06fe2b99..6b144b4f78 100644 --- a/plugins/scaffolder-common/src/TaskSpec.ts +++ b/plugins/scaffolder-common/src/TaskSpec.ts @@ -15,7 +15,7 @@ */ import type { EntityMeta, UserEntity } from '@backstage/catalog-model'; -import type { JsonObject, JsonValue } from '@backstage/types'; +import type { JsonArray, JsonObject, JsonValue } from '@backstage/types'; /** * Information about a template that is stored on a task specification. @@ -70,6 +70,10 @@ export interface TaskStep { * When this is false, or if the templated value string evaluates to something that is falsy the step will be skipped. */ if?: string | boolean; + /** + * Run step repeatedly + */ + each?: string | JsonArray; } /** diff --git a/plugins/scaffolder-node/src/actions/types.ts b/plugins/scaffolder-node/src/actions/types.ts index e35f55aa67..7e8c2b4f5f 100644 --- a/plugins/scaffolder-node/src/actions/types.ts +++ b/plugins/scaffolder-node/src/actions/types.ts @@ -71,6 +71,11 @@ export type ActionContext< * Implement the signal to make your custom step abortable https://developer.mozilla.org/en-US/docs/Web/API/AbortController/signal */ signal?: AbortSignal; + + /** + * Optional value of each invocation + */ + each?: JsonObject; }; /** @public */ From e514aac3eac0e3b730d8bef4077f84fb1fcd6303 Mon Sep 17 00:00:00 2001 From: Alex Eftimie Date: Sun, 11 Jun 2023 23:54:41 +0200 Subject: [PATCH 2/8] Add .changeset Signed-off-by: Alex Eftimie --- .changeset/tasty-lamps-shop.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/tasty-lamps-shop.md diff --git a/.changeset/tasty-lamps-shop.md b/.changeset/tasty-lamps-shop.md new file mode 100644 index 0000000000..5036932816 --- /dev/null +++ b/.changeset/tasty-lamps-shop.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-scaffolder-backend': minor +'@backstage/plugin-scaffolder-common': minor +'@backstage/plugin-scaffolder-node': minor +--- + +Introduce `each` property on action steps, allowing them to be ran repeatedly. From 0f58402454bd6a089f0a06741a3afea4cd9c0aed Mon Sep 17 00:00:00 2001 From: Alex Eftimie Date: Mon, 12 Jun 2023 00:08:07 +0200 Subject: [PATCH 3/8] Update docs. Fix tsc Signed-off-by: Alex Eftimie --- .../software-templates/writing-templates.md | 22 +++++++++++++++++++ .../tasks/NunjucksWorkflowRunner.ts | 4 ++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/docs/features/software-templates/writing-templates.md b/docs/features/software-templates/writing-templates.md index 67bcb6c179..81418faf21 100644 --- a/docs/features/software-templates/writing-templates.md +++ b/docs/features/software-templates/writing-templates.md @@ -495,6 +495,7 @@ template. These follow the same standard format: - id: fetch-base # A unique id for the step name: Fetch Base # A title displayed in the frontend if: ${{ parameters.name }} # Optional condition, skip the step if not truthy + each: ${{ parameters.iterable }} # Optional iterable, run the same step multiple times action: fetch:template # An action to call input: # Input that is passed as arguments to the action handler url: ./template @@ -506,6 +507,27 @@ By default we ship some [built in actions](./builtin-actions.md) that you can take a look at, or you can [create your own custom actions](./writing-custom-actions.md). +When `each` is provided, the current iteration value is available in the `${{ each }}` input. + +Examples: + +```yaml +each: ['apples', 'oranges'] +input: + values: + fruit: ${{ each}} +``` + +```yaml +each: [{ name: 'apple', count: 3 }, { name: 'orange', count: 1 }] +input: + values: + fruit: ${{ each.name }} + count: ${{ each.count }} +``` + +When `each` is used, the outputs of a repeated step are returned as an array of outputs from each iteration. + ## Outputs Each individual step can output some variables that can be used in the diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index e3233483ef..a6c486c58d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -25,7 +25,7 @@ import * as winston from 'winston'; import fs from 'fs-extra'; import path from 'path'; import nunjucks from 'nunjucks'; -import { JsonObject, JsonValue } from '@backstage/types'; +import { JsonArray, JsonObject, JsonValue } from '@backstage/types'; import { InputError, NotAllowedError } from '@backstage/errors'; import { PassThrough } from 'stream'; import { generateExampleOutput, isTruthy } from './helper'; @@ -351,7 +351,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { output(name: string, value: JsonValue) { if (step.each) { stepOutput[name] = stepOutput[name] || []; - stepOutput[name].push(value); + (stepOutput[name] as JsonArray).push(value); } else { stepOutput[name] = value; } From 6a754699b43e1387289fbce751224b8405b2fb61 Mon Sep 17 00:00:00 2001 From: Alex Eftimie Date: Mon, 12 Jun 2023 00:21:40 +0200 Subject: [PATCH 4/8] Update api-report Signed-off-by: Alex Eftimie --- plugins/scaffolder-common/api-report.md | 2 ++ plugins/scaffolder-node/api-report.md | 1 + 2 files changed, 3 insertions(+) diff --git a/plugins/scaffolder-common/api-report.md b/plugins/scaffolder-common/api-report.md index 7c40780b57..6113c72659 100644 --- a/plugins/scaffolder-common/api-report.md +++ b/plugins/scaffolder-common/api-report.md @@ -5,6 +5,7 @@ ```ts import { Entity } from '@backstage/catalog-model'; import type { EntityMeta } from '@backstage/catalog-model'; +import type { JsonArray } from '@backstage/types'; import { JsonObject } from '@backstage/types'; import type { JsonValue } from '@backstage/types'; import { KindValidator } from '@backstage/catalog-model'; @@ -36,6 +37,7 @@ export interface TaskSpecV1beta3 { // @public export interface TaskStep { action: string; + each?: string | JsonArray; id: string; if?: string | boolean; input?: JsonObject; diff --git a/plugins/scaffolder-node/api-report.md b/plugins/scaffolder-node/api-report.md index 46aba960d8..962daca2ad 100644 --- a/plugins/scaffolder-node/api-report.md +++ b/plugins/scaffolder-node/api-report.md @@ -36,6 +36,7 @@ export type ActionContext< ref?: string; }; signal?: AbortSignal; + each?: JsonObject; }; // @public From 17d626cad4d6191bc15138c51f87541501847471 Mon Sep 17 00:00:00 2001 From: Alex Eftimie Date: Thu, 3 Aug 2023 10:34:25 +0300 Subject: [PATCH 5/8] Fix variable passing Signed-off-by: Alex Eftimie --- .../tasks/NunjucksWorkflowRunner.test.ts | 66 ++++++++++++++++++- .../tasks/NunjucksWorkflowRunner.ts | 11 +++- 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts index 8fbbd45b94..253a6c4388 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts @@ -574,7 +574,7 @@ describe('DefaultWorkflowRunner', () => { }); describe('each', () => { - it('should run a step repeatedly', async () => { + it('should run a step repeatedly - flat values', async () => { const task = createMockTaskWithSpec({ apiVersion: 'scaffolder.backstage.io/v1beta3', steps: [ @@ -583,7 +583,7 @@ describe('DefaultWorkflowRunner', () => { name: 'name', each: '${{parameters.colors}}', action: 'jest-mock-action', - input: { color: '${{each}}' }, + input: { color: '${{each.value}}' }, }, ], output: {}, @@ -604,6 +604,68 @@ describe('DefaultWorkflowRunner', () => { expect.objectContaining({ input: { color: 'red' } }), ); }); + + it('should run a step repeatedly - object list', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'scaffolder.backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + each: '${{parameters.settings}}', + action: 'jest-mock-action', + input: { + key: '${{each.key}}', + value: '${{each.value}}', + }, + }, + ], + output: {}, + parameters: { + settings: [{ color: 'blue' }], + }, + }); + + await runner.execute(task); + + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ + input: { key: '0', value: { color: 'blue' } }, + }), + ); + }); + + it('should run a step repeatedly - object', async () => { + const task = createMockTaskWithSpec({ + apiVersion: 'scaffolder.backstage.io/v1beta3', + steps: [ + { + id: 'test', + name: 'name', + each: '${{parameters.settings}}', + action: 'jest-mock-action', + input: { key: '${{each.key}}', value: '${{each.value}}' }, + }, + ], + output: {}, + parameters: { + settings: { color: 'blue', transparent: 'yes' }, + }, + }); + + await runner.execute(task); + + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ + input: { key: 'color', value: 'blue' }, + }), + ); + expect(fakeActionHandler).toHaveBeenCalledWith( + expect.objectContaining({ + input: { key: 'transparent', value: 'yes' }, + }), + ); + }); }); describe('secrets', () => { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index a6c486c58d..b0b8fd6d09 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -315,7 +315,9 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { const iterations = new Array(); if (step.each) { const each = await this.render(step.each, context, renderTemplate); - iterations.push(...each); + iterations.push( + ...Object.keys(each).map((key: any) => [key, each[key]]), + ); } else { iterations.push({}); } @@ -324,13 +326,16 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { for (const iteration of iterations) { if (step.each) { taskLogger.info(`Running step each: ${iteration}`); - context.each = iteration; + const iterationContext = { + ...context, + each: { key: iteration[0], value: iteration[1] }, + }; // re-render input with the modified context that includes each actionInput = (step.input && this.render( step.input, - { ...context, secrets: task.secrets ?? {} }, + { ...iterationContext, secrets: task.secrets ?? {} }, renderTemplate, )) ?? {}; From 43b61fdb394c987def5b9ccb2186d71ab5c4e57e Mon Sep 17 00:00:00 2001 From: Alex Eftimie Date: Thu, 3 Aug 2023 10:44:37 +0300 Subject: [PATCH 6/8] fix tsc Signed-off-by: Alex Eftimie --- .../src/scaffolder/tasks/NunjucksWorkflowRunner.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index b0b8fd6d09..1f93ded94c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -316,7 +316,9 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { if (step.each) { const each = await this.render(step.each, context, renderTemplate); iterations.push( - ...Object.keys(each).map((key: any) => [key, each[key]]), + ...Object.keys(each).map((key: any) => { + return { key: key, value: each[key] }; + }), ); } else { iterations.push({}); @@ -328,7 +330,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { taskLogger.info(`Running step each: ${iteration}`); const iterationContext = { ...context, - each: { key: iteration[0], value: iteration[1] }, + each: iteration, }; // re-render input with the modified context that includes each actionInput = From 5b0b80f39582c2082d1a57229c66899a8b9f67f5 Mon Sep 17 00:00:00 2001 From: Alex Eftimie Date: Thu, 3 Aug 2023 12:23:46 +0300 Subject: [PATCH 7/8] fix docs Signed-off-by: Alex Eftimie --- docs/features/software-templates/writing-templates.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/features/software-templates/writing-templates.md b/docs/features/software-templates/writing-templates.md index db342bcf61..57e261f19f 100644 --- a/docs/features/software-templates/writing-templates.md +++ b/docs/features/software-templates/writing-templates.md @@ -526,8 +526,8 @@ input: each: [{ name: 'apple', count: 3 }, { name: 'orange', count: 1 }] input: values: - fruit: ${{ each.name }} - count: ${{ each.count }} + fruit: ${{ each.value.name }} + count: ${{ each.value.count }} ``` When `each` is used, the outputs of a repeated step are returned as an array of outputs from each iteration. From 027cad759b12d20b1a580ba85af2325885444647 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 11 Aug 2023 12:16:48 +0200 Subject: [PATCH 8/8] chore: fixing docs Signed-off-by: blam --- docs/features/software-templates/writing-templates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/software-templates/writing-templates.md b/docs/features/software-templates/writing-templates.md index 57e261f19f..135d508513 100644 --- a/docs/features/software-templates/writing-templates.md +++ b/docs/features/software-templates/writing-templates.md @@ -519,7 +519,7 @@ Examples: each: ['apples', 'oranges'] input: values: - fruit: ${{ each}} + fruit: ${{ each.value }} ``` ```yaml