From ff1bb4c39ecf55d801b0c33244e292b6f0949ea7 Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Mon, 1 Jul 2024 22:17:07 +0200 Subject: [PATCH 1/4] Added a documentation how to use checkpoints + the fix when to clean the workspace Signed-off-by: bnechyporenko --- .changeset/kind-dancers-boil.md | 5 ++++ .../writing-custom-actions.md | 26 ++++++++++++++++++- .../tasks/NunjucksWorkflowRunner.ts | 13 ++++++---- 3 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 .changeset/kind-dancers-boil.md diff --git a/.changeset/kind-dancers-boil.md b/.changeset/kind-dancers-boil.md new file mode 100644 index 0000000000..8e69e830df --- /dev/null +++ b/.changeset/kind-dancers-boil.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Added a documentation how to use checkpoints + the fix when to clean the workspace diff --git a/docs/features/software-templates/writing-custom-actions.md b/docs/features/software-templates/writing-custom-actions.md index e8b354bd45..41bd7d2026 100644 --- a/docs/features/software-templates/writing-custom-actions.md +++ b/docs/features/software-templates/writing-custom-actions.md @@ -145,6 +145,8 @@ When the action `handler` is called, we provide you a `context` as the only argument. It looks like the following: - `ctx.baseUrl` - a string where the template is located +- `ctx.checkpoint` - _Experimental_ allows to implement idempotency of the actions by not re-running the same function again if it was + executed successfully on the previous run. - `ctx.logger` - a Winston logger for additional logging inside your action - `ctx.logStream` - a stream version of the logger if needed - `ctx.workspacePath` - a string of the working directory of the template run @@ -153,7 +155,7 @@ argument. It looks like the following: - `ctx.output` - a function which you can call to set outputs that match the JSON schema or `zod` in `schema.output` for ex. `ctx.output('downloadUrl', myDownloadUrl)` - `createTemporaryDirectory` a function to call to give you a temporary - directory somewhere on the runner so you can store some files there rather + directory somewhere on the runner, so you can store some files there rather than polluting the `workspacePath` - `ctx.metadata` - an object containing a `name` field, indicating the template name. More metadata fields may be added later. @@ -217,6 +219,28 @@ import { }) ``` +### Using Checkpoints in Custom Actions (Experimental) + +Idempotent action could be achieved via the usage of checkpoints. + +Example: + +```ts +const res = await ctx.checkpoint?.('create.projects', async () => { + const projectStgId = createStagingProjectId(); + const projectProId = createProductionProjectId(); + + return { + projectStgId, + projectProId, + }; +}); +``` + +You have to define the unique key in scope of the scaffolder task for your checkpoint. During the execution task engine +will check if the checkpoint with such key was already executed or not, if yes, and the run was successful, the callback +will be skipped and instead the stored value will be returned. + ### Register Custom Actions with the Legacy Backend System Once you have your Custom Action ready for usage with the scaffolder, you'll diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index 3b88fd0f10..69f17e9dce 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -508,11 +508,11 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { const output = this.render(task.spec.output, context, renderTemplate); await taskTrack.markSuccessful(); + await task.cleanWorkspace?.(); return { output }; } finally { if (workspacePath) { - await task.cleanWorkspace?.(); await fs.remove(workspacePath); } } @@ -554,10 +554,13 @@ function scaffoldingTracker() { step: TaskStep, action: TemplateAction, ) { - task.emitLog(`Skipping because ${action.id} does not support dry-run`, { - stepId: step.id, - status: 'skipped', - }); + await task.emitLog( + `Skipping because ${action.id} does not support dry-run`, + { + stepId: step.id, + status: 'skipped', + }, + ); } async function markSuccessful() { From b29b39302ca0df2bb4cd97753f60f52e6470d747 Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 2 Jul 2024 09:08:29 +0200 Subject: [PATCH 2/4] Excluding the fix from this PR Signed-off-by: bnechyporenko --- .changeset/kind-dancers-boil.md | 2 +- .../src/scaffolder/tasks/NunjucksWorkflowRunner.ts | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.changeset/kind-dancers-boil.md b/.changeset/kind-dancers-boil.md index 8e69e830df..4451cab4f4 100644 --- a/.changeset/kind-dancers-boil.md +++ b/.changeset/kind-dancers-boil.md @@ -2,4 +2,4 @@ '@backstage/plugin-scaffolder-backend': patch --- -Added a documentation how to use checkpoints + the fix when to clean the workspace +Added a documentation how to use checkpoints diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index 69f17e9dce..3b88fd0f10 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -508,11 +508,11 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { const output = this.render(task.spec.output, context, renderTemplate); await taskTrack.markSuccessful(); - await task.cleanWorkspace?.(); return { output }; } finally { if (workspacePath) { + await task.cleanWorkspace?.(); await fs.remove(workspacePath); } } @@ -554,13 +554,10 @@ function scaffoldingTracker() { step: TaskStep, action: TemplateAction, ) { - await task.emitLog( - `Skipping because ${action.id} does not support dry-run`, - { - stepId: step.id, - status: 'skipped', - }, - ); + task.emitLog(`Skipping because ${action.id} does not support dry-run`, { + stepId: step.id, + status: 'skipped', + }); } async function markSuccessful() { From 9182cb074f2933291ef72443b43c5d386db1093d Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 2 Jul 2024 19:19:56 +0200 Subject: [PATCH 3/4] Feedback Signed-off-by: bnechyporenko --- docs/features/software-templates/writing-custom-actions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/features/software-templates/writing-custom-actions.md b/docs/features/software-templates/writing-custom-actions.md index 41bd7d2026..3b5843e795 100644 --- a/docs/features/software-templates/writing-custom-actions.md +++ b/docs/features/software-templates/writing-custom-actions.md @@ -146,7 +146,7 @@ argument. It looks like the following: - `ctx.baseUrl` - a string where the template is located - `ctx.checkpoint` - _Experimental_ allows to implement idempotency of the actions by not re-running the same function again if it was - executed successfully on the previous run. + executed successfully on the previous run. More information [here](https://github.com/backstage/backstage/tree/master/beps/0004-scaffolder-task-idempotency#checkpoints) - `ctx.logger` - a Winston logger for additional logging inside your action - `ctx.logStream` - a stream version of the logger if needed - `ctx.workspacePath` - a string of the working directory of the template run @@ -225,7 +225,7 @@ Idempotent action could be achieved via the usage of checkpoints. Example: -```ts +```ts title="plugins/my-company-scaffolder-actions-plugin/src/vendor/my-custom-action.ts" const res = await ctx.checkpoint?.('create.projects', async () => { const projectStgId = createStagingProjectId(); const projectProId = createProductionProjectId(); From 920d0f9c96c0e92275f5566a6a9e495b03aa1f12 Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 2 Jul 2024 19:20:54 +0200 Subject: [PATCH 4/4] Feedback Signed-off-by: bnechyporenko --- docs/features/software-templates/writing-custom-actions.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/features/software-templates/writing-custom-actions.md b/docs/features/software-templates/writing-custom-actions.md index 3b5843e795..6c1ea573db 100644 --- a/docs/features/software-templates/writing-custom-actions.md +++ b/docs/features/software-templates/writing-custom-actions.md @@ -145,8 +145,10 @@ When the action `handler` is called, we provide you a `context` as the only argument. It looks like the following: - `ctx.baseUrl` - a string where the template is located -- `ctx.checkpoint` - _Experimental_ allows to implement idempotency of the actions by not re-running the same function again if it was - executed successfully on the previous run. More information [here](https://github.com/backstage/backstage/tree/master/beps/0004-scaffolder-task-idempotency#checkpoints) +- `ctx.checkpoint` - _Experimental_ allows to + implement [idempotency of the actions](https://github.com/backstage/backstage/tree/master/beps/0004-scaffolder-task-idempotency) + by not re-running the same function again if it was + executed successfully on the previous run. - `ctx.logger` - a Winston logger for additional logging inside your action - `ctx.logStream` - a stream version of the logger if needed - `ctx.workspacePath` - a string of the working directory of the template run