Checkpoint

Signed-off-by: bnechyporenko <bnechyporenko@bol.com>

plugins/scaffolder-backend-module-github/package.json

@@ -37,7 +37,9 @@
     "@backstage/config": "workspace:^",
     "@backstage/errors": "workspace:^",
     "@backstage/integration": "workspace:^",
+    "@backstage/plugin-scaffolder-backend": "workspace:^",
     "@backstage/plugin-scaffolder-node": "workspace:^",
+    "@backstage/types": "workspace:^",
     "@octokit/webhooks": "^10.0.0",
     "libsodium-wrappers": "^0.7.11",
     "octokit": "^3.0.0",

plugins/scaffolder-backend-module-github/src/actions/github.ts

@@ -244,6 +244,7 @@ export function createPublishGithubAction(options: {
         repoVariables,
         secrets,
         ctx.logger,
+        ctx.checkpoint,
       );
 
       const remoteUrl = newRepo.clone_url;

plugins/scaffolder-backend-module-github/src/actions/helpers.ts

@@ -36,6 +36,8 @@ import {
   enableBranchProtectionOnDefaultRepoBranch,
   entityRefToName,
 } from './gitHelpers';
+import { JsonObject } from '@backstage/types';
+import { defineCheckpoint } from '@backstage/plugin-scaffolder-backend';
 
 const DEFAULT_TIMEOUT_MS = 60_000;
 
@@ -138,6 +140,10 @@ export async function createGithubRepoWithCollaboratorsAndTopics(
   repoVariables: { [key: string]: string } | undefined,
   secrets: { [key: string]: string } | undefined,
   logger: Logger,
+  checkpoint?: <U extends JsonObject>(
+    key: string,
+    fn: () => Promise<U>,
+  ) => Promise<U>,
 ) {
   // eslint-disable-next-line testing-library/no-await-sync-queries
   const user = await client.rest.users.getByUsername({
@@ -148,59 +154,70 @@ export async function createGithubRepoWithCollaboratorsAndTopics(
     await validateAccessTeam(client, access);
   }
 
-  const repoCreationPromise =
-    user.data.type === 'Organization'
-      ? client.rest.repos.createInOrg({
-          name: repo,
-          org: owner,
-          private: repoVisibility === 'private',
-          // @ts-ignore https://github.com/octokit/types.ts/issues/522
-          visibility: repoVisibility,
-          description: description,
-          delete_branch_on_merge: deleteBranchOnMerge,
-          allow_merge_commit: allowMergeCommit,
-          allow_squash_merge: allowSquashMerge,
-          squash_merge_commit_title: squashMergeCommitTitle,
-          squash_merge_commit_message: squashMergeCommitMessage,
-          allow_rebase_merge: allowRebaseMerge,
-          allow_auto_merge: allowAutoMerge,
-          homepage: homepage,
-          has_projects: hasProjects,
-          has_wiki: hasWiki,
-          has_issues: hasIssues,
-        })
-      : client.rest.repos.createForAuthenticatedUser({
-          name: repo,
-          private: repoVisibility === 'private',
-          description: description,
-          delete_branch_on_merge: deleteBranchOnMerge,
-          allow_merge_commit: allowMergeCommit,
-          allow_squash_merge: allowSquashMerge,
-          squash_merge_commit_title: squashMergeCommitTitle,
-          squash_merge_commit_message: squashMergeCommitMessage,
-          allow_rebase_merge: allowRebaseMerge,
-          allow_auto_merge: allowAutoMerge,
-          homepage: homepage,
-          has_projects: hasProjects,
-          has_wiki: hasWiki,
-          has_issues: hasIssues,
-        });
+  const repoCreation = async () => {
+    const repoCreationPromise =
+      user.data.type === 'Organization'
+        ? client.rest.repos.createInOrg({
+            name: repo,
+            org: owner,
+            private: repoVisibility === 'private',
+            // @ts-ignore https://github.com/octokit/types.ts/issues/522
+            visibility: repoVisibility,
+            description: description,
+            delete_branch_on_merge: deleteBranchOnMerge,
+            allow_merge_commit: allowMergeCommit,
+            allow_squash_merge: allowSquashMerge,
+            squash_merge_commit_title: squashMergeCommitTitle,
+            squash_merge_commit_message: squashMergeCommitMessage,
+            allow_rebase_merge: allowRebaseMerge,
+            allow_auto_merge: allowAutoMerge,
+            homepage: homepage,
+            has_projects: hasProjects,
+            has_wiki: hasWiki,
+            has_issues: hasIssues,
+          })
+        : client.rest.repos.createForAuthenticatedUser({
+            name: repo,
+            private: repoVisibility === 'private',
+            description: description,
+            delete_branch_on_merge: deleteBranchOnMerge,
+            allow_merge_commit: allowMergeCommit,
+            allow_squash_merge: allowSquashMerge,
+            squash_merge_commit_title: squashMergeCommitTitle,
+            squash_merge_commit_message: squashMergeCommitMessage,
+            allow_rebase_merge: allowRebaseMerge,
+            allow_auto_merge: allowAutoMerge,
+            homepage: homepage,
+            has_projects: hasProjects,
+            has_wiki: hasWiki,
+            has_issues: hasIssues,
+          });
 
-  let newRepo;
+    let newRepo;
 
-  try {
-    newRepo = (await repoCreationPromise).data;
-  } catch (e) {
-    assertError(e);
-    if (e.message === 'Resource not accessible by integration') {
-      logger.warn(
-        `The GitHub app or token provided may not have the required permissions to create the ${user.data.type} repository ${owner}/${repo}.`,
+    try {
+      newRepo = (await repoCreationPromise).data;
+    } catch (e) {
+      assertError(e);
+      if (e.message === 'Resource not accessible by integration') {
+        logger.warn(
+          `The GitHub app or token provided may not have the required permissions to create the ${user.data.type} repository ${owner}/${repo}.`,
+        );
+      }
+      throw new Error(
+        `Failed to create the ${user.data.type} repository ${owner}/${repo}, ${e.message}`,
       );
     }
-    throw new Error(
-      `Failed to create the ${user.data.type} repository ${owner}/${repo}, ${e.message}`,
-    );
-  }
+    return { newRepo };
+  };
+
+  const { newRepo } = await defineCheckpoint<{
+    newRepo: { clone_url: string; html_url: string };
+  }>({
+    key: 'v1.task.checkpoint.repo.creation',
+    checkpoint,
+    fn: repoCreation,
+  });
 
   if (access?.startsWith(`${owner}/`)) {
     const [, team] = access.split('/');
@@ -213,11 +230,19 @@ export async function createGithubRepoWithCollaboratorsAndTopics(
     });
     // No need to add access if it's the person who owns the personal account
   } else if (access && access !== owner) {
-    await client.rest.repos.addCollaborator({
-      owner,
-      repo,
-      username: access,
-      permission: 'admin',
+    const addCollaborator = async () => {
+      await client.rest.repos.addCollaborator({
+        owner,
+        repo,
+        username: access,
+        permission: 'admin',
+      });
+      return {};
+    };
+    await defineCheckpoint({
+      key: 'v1.task.checkpoint.add.collaborator',
+      checkpoint,
+      fn: addCollaborator,
     });
   }
 
@@ -225,11 +250,19 @@ export async function createGithubRepoWithCollaboratorsAndTopics(
     for (const collaborator of collaborators) {
       try {
         if ('user' in collaborator) {
-          await client.rest.repos.addCollaborator({
-            owner,
-            repo,
-            username: entityRefToName(collaborator.user),
-            permission: collaborator.access,
+          const addCollaborator = async () => {
+            await client.rest.repos.addCollaborator({
+              owner,
+              repo,
+              username: entityRefToName(collaborator.user),
+              permission: collaborator.access,
+            });
+            return {};
+          };
+          await defineCheckpoint({
+            key: `v1.task.checkpoint.add.collaborator.${collaborator.user}`,
+            checkpoint,
+            fn: addCollaborator,
           });
         } else if ('team' in collaborator) {
           await client.rest.teams.addOrUpdateRepoPermissionsInOrg({
@@ -264,11 +297,19 @@ export async function createGithubRepoWithCollaboratorsAndTopics(
   }
 
   for (const [key, value] of Object.entries(repoVariables ?? {})) {
-    await client.rest.actions.createRepoVariable({
-      owner,
-      repo,
-      name: key,
-      value: value,
+    const createRepoVariable = async () => {
+      await client.rest.actions.createRepoVariable({
+        owner,
+        repo,
+        name: key,
+        value: value,
+      });
+      return {};
+    };
+    await defineCheckpoint({
+      key: `v1.task.checkpoint.create.repo.variable.${key}`,
+      checkpoint,
+      fn: createRepoVariable,
     });
   }
 

plugins/scaffolder-backend/migrations/20240203232000_state.js

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.up = async function up(knex) {
+  await knex.schema.alterTable('tasks', table => {
+    table.text('state').nullable().comment('A state of the checkpoints');
+  });
+};
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.down = async function down(knex) {
+  await knex.schema.alterTable('tasks', table => {
+    table.dropColumn('state');
+  });
+};

plugins/scaffolder-backend/src/index.ts

@@ -23,5 +23,6 @@
 export * from './scaffolder';
 export * from './service/router';
 export * from './lib';
+export { defineCheckpoint } from './util/defineCheckpoint';
 
 export * from './deprecated';

plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts

@@ -30,6 +30,7 @@ import {
   TaskStoreCreateTaskResult,
   TaskStoreShutDownTaskOptions,
   TaskStoreRecoverTaskOptions,
+  TaskStoreStateOptions,
 } from './types';
 import {
   SerializedTaskEvent,
@@ -52,6 +53,7 @@ export type RawDbTaskRow = {
   id: string;
   spec: string;
   status: TaskStatus;
+  state?: string;
   last_heartbeat_at?: string;
   created_at: string;
   created_by: string | null;
@@ -394,6 +396,17 @@ export class DatabaseTaskStore implements TaskStore {
     });
   }
 
+  async saveCheckpoint?(options: TaskStoreStateOptions): Promise<void> {
+    if (options.state) {
+      const serializedState = JSON.stringify(options.state);
+      await this.db<RawDbTaskRow>('tasks')
+        .where({ id: options.taskId })
+        .update({
+          state: serializedState,
+        });
+    }
+  }
+
   async listEvents(
     options: TaskStoreListEventsOptions,
   ): Promise<{ events: SerializedTaskEvent[] }> {

plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts

@@ -349,6 +349,14 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
           logger: taskLogger,
           logStream: streamLogger,
           workspacePath,
+          async checkpoint<U extends JsonObject>(
+            key: string,
+            fn: () => Promise<U>,
+          ) {
+            const value = await fn();
+            task.updateCheckpoint?.(key, value);
+            return value;
+          },
           createTemporaryDirectory: async () => {
             const tmpDir = await fs.mkdtemp(
               `${workspacePath}_step-${step.id}-`,

plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts

@@ -16,7 +16,7 @@
 
 import { Config } from '@backstage/config';
 import { TaskSpec } from '@backstage/plugin-scaffolder-common';
-import { TaskSecrets } from '@backstage/plugin-scaffolder-node';
+import { TaskSecrets, TaskState } from '@backstage/plugin-scaffolder-node';
 import { JsonObject, Observable } from '@backstage/types';
 import { Logger } from 'winston';
 import ObservableImpl from 'zen-observable';
@@ -91,6 +91,14 @@ export class TaskManager implements TaskContext {
     });
   }
 
+  async updateCheckpoint?(key: string, value: JsonObject): Promise<void> {
+    this.task.state = { [key]: value };
+    await this.storage.saveCheckpoint?.({
+      taskId: this.task.taskId,
+      state: this.task.state,
+    });
+  }
+
   async complete(
     result: TaskCompletionState,
     metadata?: JsonObject,
@@ -144,6 +152,10 @@ export interface CurrentClaimedTask {
    * The secrets that are stored with the task.
    */
   secrets?: TaskSecrets;
+  /**
+   * The state of checkpoints of the task.
+   */
+  state?: TaskState;
   /**
    * The creator of the task.
    */

plugins/scaffolder-backend/src/scaffolder/tasks/types.ts

@@ -16,7 +16,7 @@
 
 import { JsonValue, JsonObject, HumanDuration } from '@backstage/types';
 import { TaskSpec, TaskStep } from '@backstage/plugin-scaffolder-common';
-import { TaskSecrets } from '@backstage/plugin-scaffolder-node';
+import { TaskSecrets, TaskState } from '@backstage/plugin-scaffolder-node';
 import {
   TemplateAction,
   TaskStatus as _TaskStatus,
@@ -113,6 +113,11 @@ export type TaskStoreEmitOptions<TBody = JsonObject> = {
   body: TBody;
 };
 
+export type TaskStoreStateOptions = {
+  taskId: string;
+  state?: TaskState;
+};
+
 /**
  * TaskStoreListEventsOptions
  *
@@ -194,6 +199,8 @@ export interface TaskStore {
 
   emitLogEvent(options: TaskStoreEmitOptions): Promise<void>;
 
+  saveCheckpoint?(options: TaskStoreStateOptions): Promise<void>;
+
   listEvents(
     options: TaskStoreListEventsOptions,
   ): Promise<{ events: SerializedTaskEvent[] }>;

plugins/scaffolder-backend/src/util/defineCheckpoint.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { JsonObject } from '@backstage/types';
+
+export type DefineCheckpointProps<U> = {
+  checkpoint?: (key: string, fn: () => Promise<U>) => Promise<U>;
+  key: string;
+  fn: () => Promise<U>;
+};
+
+export const defineCheckpoint = async <U extends JsonObject>(
+  props: DefineCheckpointProps<U>,
+): Promise<U> => {
+  const { checkpoint, fn, key } = props;
+  return checkpoint
+    ? checkpoint?.(key, async () => {
+        return await fn();
+      })
+    : fn();
+};

plugins/scaffolder-node/src/actions/types.ts

@@ -35,6 +35,10 @@ export type ActionContext<
   secrets?: TaskSecrets;
   workspacePath: string;
   input: TActionInput;
+  checkpoint?<U extends JsonObject>(
+    key: string,
+    fn: () => Promise<U>,
+  ): Promise<U>;
   output(
     name: keyof TActionOutput,
     value: TActionOutput[keyof TActionOutput],

plugins/scaffolder-node/src/tasks/index.ts

@@ -24,5 +24,6 @@ export type {
   TaskCompletionState,
   TaskContext,
   TaskEventType,
+  TaskState,
   TaskStatus,
 } from './types';

plugins/scaffolder-node/src/tasks/types.ts

@@ -26,6 +26,13 @@ export type TaskSecrets = Record<string, string> & {
   backstageToken?: string;
 };
 
+/**
+ * TaskState
+ *
+ * @public
+ */
+export type TaskState = Record<string, JsonObject>;
+
 /**
  * The status of each step of the Task
  *
@@ -110,6 +117,7 @@ export interface TaskContext {
   cancelSignal: AbortSignal;
   spec: TaskSpec;
   secrets?: TaskSecrets;
+  state?: TaskState;
   createdBy?: string;
   done: boolean;
   isDryRun?: boolean;
@@ -118,6 +126,8 @@ export interface TaskContext {
 
   emitLog(message: string, logMetadata?: JsonObject): Promise<void>;
 
+  updateCheckpoint?(key: string, value: JsonObject): Promise<void>;
+
   getWorkspaceName(): Promise<string>;
 }
 

yarn.lock

@@ -8354,7 +8354,9 @@ __metadata:
     "@backstage/config": "workspace:^"
     "@backstage/errors": "workspace:^"
     "@backstage/integration": "workspace:^"
+    "@backstage/plugin-scaffolder-backend": "workspace:^"
     "@backstage/plugin-scaffolder-node": "workspace:^"
+    "@backstage/types": "workspace:^"
     "@octokit/webhooks": ^10.0.0
     "@types/libsodium-wrappers": ^0.7.10
     fs-extra: 10.1.0