Merge pull request #18157 from alexef/scaffolder-each

scaffolder: enable each property on task step (#8890)

.changeset/tasty-lamps-shop.md

@@ -0,0 +1,7 @@
+---
+'@backstage/plugin-scaffolder-backend': minor
+'@backstage/plugin-scaffolder-common': minor
+'@backstage/plugin-scaffolder-node': minor
+---
+
+Introduce `each` property on action steps, allowing them to be ran repeatedly.

docs/features/software-templates/writing-templates.md

@@ -499,6 +499,7 @@ template. These follow the same standard format:
 - id: fetch-base # A unique id for the step
   name: Fetch Base # A title displayed in the frontend
   if: ${{ parameters.name }} # Optional condition, skip the step if not truthy
+  each: ${{ parameters.iterable }} # Optional iterable, run the same step multiple times
   action: fetch:template # An action to call
   input: # Input that is passed as arguments to the action handler
     url: ./template
@@ -510,6 +511,27 @@ By default we ship some [built in actions](./builtin-actions.md) that you can
 take a look at, or you can
 [create your own custom actions](./writing-custom-actions.md).
 
+When `each` is provided, the current iteration value is available in the `${{ each }}` input.
+
+Examples:
+
+```yaml
+each: ['apples', 'oranges']
+input:
+  values:
+    fruit: ${{ each.value }}
+```
+
+```yaml
+each: [{ name: 'apple', count: 3 }, { name: 'orange', count: 1 }]
+input:
+  values:
+    fruit: ${{ each.value.name }}
+    count: ${{ each.value.count }}
+```
+
+When `each` is used, the outputs of a repeated step are returned as an array of outputs from each iteration.
+
 ## Outputs
 
 Each individual step can output some variables that can be used in the

plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.test.ts

@@ -573,6 +573,101 @@ describe('DefaultWorkflowRunner', () => {
     });
   });
 
+  describe('each', () => {
+    it('should run a step repeatedly - flat values', async () => {
+      const task = createMockTaskWithSpec({
+        apiVersion: 'scaffolder.backstage.io/v1beta3',
+        steps: [
+          {
+            id: 'test',
+            name: 'name',
+            each: '${{parameters.colors}}',
+            action: 'jest-mock-action',
+            input: { color: '${{each.value}}' },
+          },
+        ],
+        output: {},
+        parameters: {
+          colors: ['blue', 'green', 'red'],
+        },
+      });
+
+      await runner.execute(task);
+
+      expect(fakeActionHandler).toHaveBeenCalledWith(
+        expect.objectContaining({ input: { color: 'blue' } }),
+      );
+      expect(fakeActionHandler).toHaveBeenCalledWith(
+        expect.objectContaining({ input: { color: 'green' } }),
+      );
+      expect(fakeActionHandler).toHaveBeenCalledWith(
+        expect.objectContaining({ input: { color: 'red' } }),
+      );
+    });
+
+    it('should run a step repeatedly - object list', async () => {
+      const task = createMockTaskWithSpec({
+        apiVersion: 'scaffolder.backstage.io/v1beta3',
+        steps: [
+          {
+            id: 'test',
+            name: 'name',
+            each: '${{parameters.settings}}',
+            action: 'jest-mock-action',
+            input: {
+              key: '${{each.key}}',
+              value: '${{each.value}}',
+            },
+          },
+        ],
+        output: {},
+        parameters: {
+          settings: [{ color: 'blue' }],
+        },
+      });
+
+      await runner.execute(task);
+
+      expect(fakeActionHandler).toHaveBeenCalledWith(
+        expect.objectContaining({
+          input: { key: '0', value: { color: 'blue' } },
+        }),
+      );
+    });
+
+    it('should run a step repeatedly - object', async () => {
+      const task = createMockTaskWithSpec({
+        apiVersion: 'scaffolder.backstage.io/v1beta3',
+        steps: [
+          {
+            id: 'test',
+            name: 'name',
+            each: '${{parameters.settings}}',
+            action: 'jest-mock-action',
+            input: { key: '${{each.key}}', value: '${{each.value}}' },
+          },
+        ],
+        output: {},
+        parameters: {
+          settings: { color: 'blue', transparent: 'yes' },
+        },
+      });
+
+      await runner.execute(task);
+
+      expect(fakeActionHandler).toHaveBeenCalledWith(
+        expect.objectContaining({
+          input: { key: 'color', value: 'blue' },
+        }),
+      );
+      expect(fakeActionHandler).toHaveBeenCalledWith(
+        expect.objectContaining({
+          input: { key: 'transparent', value: 'yes' },
+        }),
+      );
+    });
+  });
+
   describe('secrets', () => {
     it('should pass through the secrets to the context', async () => {
       const task = createMockTaskWithSpec(

plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts

@@ -25,7 +25,7 @@ import * as winston from 'winston';
 import fs from 'fs-extra';
 import path from 'path';
 import nunjucks from 'nunjucks';
-import { JsonObject, JsonValue } from '@backstage/types';
+import { JsonArray, JsonObject, JsonValue } from '@backstage/types';
 import { InputError, NotAllowedError } from '@backstage/errors';
 import { PassThrough } from 'stream';
 import { generateExampleOutput, isTruthy } from './helper';
@@ -76,6 +76,7 @@ type TemplateContext = {
     entity?: UserEntity;
     ref?: string;
   };
+  each?: JsonValue;
 };
 
 const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta3 => {
@@ -311,25 +312,63 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
       const tmpDirs = new Array<string>();
       const stepOutput: { [outputName: string]: JsonValue } = {};
 
-      await action.handler({
-        input,
-        secrets: task.secrets ?? {},
-        logger: taskLogger,
-        logStream: streamLogger,
-        workspacePath,
-        createTemporaryDirectory: async () => {
-          const tmpDir = await fs.mkdtemp(`${workspacePath}_step-${step.id}-`);
-          tmpDirs.push(tmpDir);
-          return tmpDir;
-        },
-        output(name: string, value: JsonValue) {
-          stepOutput[name] = value;
-        },
-        templateInfo: task.spec.templateInfo,
-        user: task.spec.user,
-        isDryRun: task.isDryRun,
-        signal: task.cancelSignal,
-      });
+      const iterations = new Array<JsonValue>();
+      if (step.each) {
+        const each = await this.render(step.each, context, renderTemplate);
+        iterations.push(
+          ...Object.keys(each).map((key: any) => {
+            return { key: key, value: each[key] };
+          }),
+        );
+      } else {
+        iterations.push({});
+      }
+
+      let actionInput = input;
+      for (const iteration of iterations) {
+        if (step.each) {
+          taskLogger.info(`Running step each: ${iteration}`);
+          const iterationContext = {
+            ...context,
+            each: iteration,
+          };
+          // re-render input with the modified context that includes each
+          actionInput =
+            (step.input &&
+              this.render(
+                step.input,
+                { ...iterationContext, secrets: task.secrets ?? {} },
+                renderTemplate,
+              )) ??
+            {};
+        }
+        await action.handler({
+          input: actionInput,
+          secrets: task.secrets ?? {},
+          logger: taskLogger,
+          logStream: streamLogger,
+          workspacePath,
+          createTemporaryDirectory: async () => {
+            const tmpDir = await fs.mkdtemp(
+              `${workspacePath}_step-${step.id}-`,
+            );
+            tmpDirs.push(tmpDir);
+            return tmpDir;
+          },
+          output(name: string, value: JsonValue) {
+            if (step.each) {
+              stepOutput[name] = stepOutput[name] || [];
+              (stepOutput[name] as JsonArray).push(value);
+            } else {
+              stepOutput[name] = value;
+            }
+          },
+          templateInfo: task.spec.templateInfo,
+          user: task.spec.user,
+          isDryRun: task.isDryRun,
+          signal: task.cancelSignal,
+        });
+      }
 
       // Remove all temporary directories that were created when executing the action
       for (const tmpDir of tmpDirs) {

plugins/scaffolder-common/api-report.md

@@ -5,6 +5,7 @@
 ```ts
 import { Entity } from '@backstage/catalog-model';
 import type { EntityMeta } from '@backstage/catalog-model';
+import type { JsonArray } from '@backstage/types';
 import { JsonObject } from '@backstage/types';
 import type { JsonValue } from '@backstage/types';
 import { KindValidator } from '@backstage/catalog-model';
@@ -36,6 +37,7 @@ export interface TaskSpecV1beta3 {
 // @public
 export interface TaskStep {
   action: string;
+  each?: string | JsonArray;
   id: string;
   if?: string | boolean;
   input?: JsonObject;

plugins/scaffolder-common/src/TaskSpec.ts

@@ -15,7 +15,7 @@
  */
 
 import type { EntityMeta, UserEntity } from '@backstage/catalog-model';
-import type { JsonObject, JsonValue } from '@backstage/types';
+import type { JsonArray, JsonObject, JsonValue } from '@backstage/types';
 
 /**
  * Information about a template that is stored on a task specification.
@@ -70,6 +70,10 @@ export interface TaskStep {
    * When this is false, or if the templated value string evaluates to something that is falsy the step will be skipped.
    */
   if?: string | boolean;
+  /**
+   * Run step repeatedly
+   */
+  each?: string | JsonArray;
 }
 
 /**

plugins/scaffolder-node/api-report.md

@@ -39,6 +39,7 @@ export type ActionContext<
     ref?: string;
   };
   signal?: AbortSignal;
+  each?: JsonObject;
 };
 
 // @public

plugins/scaffolder-node/src/actions/types.ts

@@ -71,6 +71,11 @@ export type ActionContext<
    * Implement the signal to make your custom step abortable https://developer.mozilla.org/en-US/docs/Web/API/AbortController/signal
    */
   signal?: AbortSignal;
+
+  /**
+   * Optional value of each invocation
+   */
+  each?: JsonObject;
 };
 
 /** @public */