diff --git a/plugins/scaffolder-backend-module-github/package.json b/plugins/scaffolder-backend-module-github/package.json index a0cccb536c..f48feaaae5 100644 --- a/plugins/scaffolder-backend-module-github/package.json +++ b/plugins/scaffolder-backend-module-github/package.json @@ -37,7 +37,9 @@ "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", + "@backstage/plugin-scaffolder-backend": "workspace:^", "@backstage/plugin-scaffolder-node": "workspace:^", + "@backstage/types": "workspace:^", "@octokit/webhooks": "^10.0.0", "libsodium-wrappers": "^0.7.11", "octokit": "^3.0.0", diff --git a/plugins/scaffolder-backend-module-github/src/actions/github.ts b/plugins/scaffolder-backend-module-github/src/actions/github.ts index 91d7a34aea..ed55714852 100644 --- a/plugins/scaffolder-backend-module-github/src/actions/github.ts +++ b/plugins/scaffolder-backend-module-github/src/actions/github.ts @@ -244,6 +244,7 @@ export function createPublishGithubAction(options: { repoVariables, secrets, ctx.logger, + ctx.checkpoint, ); const remoteUrl = newRepo.clone_url; diff --git a/plugins/scaffolder-backend-module-github/src/actions/helpers.ts b/plugins/scaffolder-backend-module-github/src/actions/helpers.ts index 4d95a6494f..87c125eda2 100644 --- a/plugins/scaffolder-backend-module-github/src/actions/helpers.ts +++ b/plugins/scaffolder-backend-module-github/src/actions/helpers.ts @@ -36,6 +36,8 @@ import { enableBranchProtectionOnDefaultRepoBranch, entityRefToName, } from './gitHelpers'; +import { JsonObject } from '@backstage/types'; +import { defineCheckpoint } from '@backstage/plugin-scaffolder-backend'; const DEFAULT_TIMEOUT_MS = 60_000; @@ -138,6 +140,10 @@ export async function createGithubRepoWithCollaboratorsAndTopics( repoVariables: { [key: string]: string } | undefined, secrets: { [key: string]: string } | undefined, logger: Logger, + checkpoint?: ( + key: string, + fn: () => Promise, + ) => Promise, ) { // eslint-disable-next-line testing-library/no-await-sync-queries const user = await client.rest.users.getByUsername({ @@ -148,59 +154,70 @@ export async function createGithubRepoWithCollaboratorsAndTopics( await validateAccessTeam(client, access); } - const repoCreationPromise = - user.data.type === 'Organization' - ? client.rest.repos.createInOrg({ - name: repo, - org: owner, - private: repoVisibility === 'private', - // @ts-ignore https://github.com/octokit/types.ts/issues/522 - visibility: repoVisibility, - description: description, - delete_branch_on_merge: deleteBranchOnMerge, - allow_merge_commit: allowMergeCommit, - allow_squash_merge: allowSquashMerge, - squash_merge_commit_title: squashMergeCommitTitle, - squash_merge_commit_message: squashMergeCommitMessage, - allow_rebase_merge: allowRebaseMerge, - allow_auto_merge: allowAutoMerge, - homepage: homepage, - has_projects: hasProjects, - has_wiki: hasWiki, - has_issues: hasIssues, - }) - : client.rest.repos.createForAuthenticatedUser({ - name: repo, - private: repoVisibility === 'private', - description: description, - delete_branch_on_merge: deleteBranchOnMerge, - allow_merge_commit: allowMergeCommit, - allow_squash_merge: allowSquashMerge, - squash_merge_commit_title: squashMergeCommitTitle, - squash_merge_commit_message: squashMergeCommitMessage, - allow_rebase_merge: allowRebaseMerge, - allow_auto_merge: allowAutoMerge, - homepage: homepage, - has_projects: hasProjects, - has_wiki: hasWiki, - has_issues: hasIssues, - }); + const repoCreation = async () => { + const repoCreationPromise = + user.data.type === 'Organization' + ? client.rest.repos.createInOrg({ + name: repo, + org: owner, + private: repoVisibility === 'private', + // @ts-ignore https://github.com/octokit/types.ts/issues/522 + visibility: repoVisibility, + description: description, + delete_branch_on_merge: deleteBranchOnMerge, + allow_merge_commit: allowMergeCommit, + allow_squash_merge: allowSquashMerge, + squash_merge_commit_title: squashMergeCommitTitle, + squash_merge_commit_message: squashMergeCommitMessage, + allow_rebase_merge: allowRebaseMerge, + allow_auto_merge: allowAutoMerge, + homepage: homepage, + has_projects: hasProjects, + has_wiki: hasWiki, + has_issues: hasIssues, + }) + : client.rest.repos.createForAuthenticatedUser({ + name: repo, + private: repoVisibility === 'private', + description: description, + delete_branch_on_merge: deleteBranchOnMerge, + allow_merge_commit: allowMergeCommit, + allow_squash_merge: allowSquashMerge, + squash_merge_commit_title: squashMergeCommitTitle, + squash_merge_commit_message: squashMergeCommitMessage, + allow_rebase_merge: allowRebaseMerge, + allow_auto_merge: allowAutoMerge, + homepage: homepage, + has_projects: hasProjects, + has_wiki: hasWiki, + has_issues: hasIssues, + }); - let newRepo; + let newRepo; - try { - newRepo = (await repoCreationPromise).data; - } catch (e) { - assertError(e); - if (e.message === 'Resource not accessible by integration') { - logger.warn( - `The GitHub app or token provided may not have the required permissions to create the ${user.data.type} repository ${owner}/${repo}.`, + try { + newRepo = (await repoCreationPromise).data; + } catch (e) { + assertError(e); + if (e.message === 'Resource not accessible by integration') { + logger.warn( + `The GitHub app or token provided may not have the required permissions to create the ${user.data.type} repository ${owner}/${repo}.`, + ); + } + throw new Error( + `Failed to create the ${user.data.type} repository ${owner}/${repo}, ${e.message}`, ); } - throw new Error( - `Failed to create the ${user.data.type} repository ${owner}/${repo}, ${e.message}`, - ); - } + return { newRepo }; + }; + + const { newRepo } = await defineCheckpoint<{ + newRepo: { clone_url: string; html_url: string }; + }>({ + key: 'v1.task.checkpoint.repo.creation', + checkpoint, + fn: repoCreation, + }); if (access?.startsWith(`${owner}/`)) { const [, team] = access.split('/'); @@ -213,11 +230,19 @@ export async function createGithubRepoWithCollaboratorsAndTopics( }); // No need to add access if it's the person who owns the personal account } else if (access && access !== owner) { - await client.rest.repos.addCollaborator({ - owner, - repo, - username: access, - permission: 'admin', + const addCollaborator = async () => { + await client.rest.repos.addCollaborator({ + owner, + repo, + username: access, + permission: 'admin', + }); + return {}; + }; + await defineCheckpoint({ + key: 'v1.task.checkpoint.add.collaborator', + checkpoint, + fn: addCollaborator, }); } @@ -225,11 +250,19 @@ export async function createGithubRepoWithCollaboratorsAndTopics( for (const collaborator of collaborators) { try { if ('user' in collaborator) { - await client.rest.repos.addCollaborator({ - owner, - repo, - username: entityRefToName(collaborator.user), - permission: collaborator.access, + const addCollaborator = async () => { + await client.rest.repos.addCollaborator({ + owner, + repo, + username: entityRefToName(collaborator.user), + permission: collaborator.access, + }); + return {}; + }; + await defineCheckpoint({ + key: `v1.task.checkpoint.add.collaborator.${collaborator.user}`, + checkpoint, + fn: addCollaborator, }); } else if ('team' in collaborator) { await client.rest.teams.addOrUpdateRepoPermissionsInOrg({ @@ -264,11 +297,19 @@ export async function createGithubRepoWithCollaboratorsAndTopics( } for (const [key, value] of Object.entries(repoVariables ?? {})) { - await client.rest.actions.createRepoVariable({ - owner, - repo, - name: key, - value: value, + const createRepoVariable = async () => { + await client.rest.actions.createRepoVariable({ + owner, + repo, + name: key, + value: value, + }); + return {}; + }; + await defineCheckpoint({ + key: `v1.task.checkpoint.create.repo.variable.${key}`, + checkpoint, + fn: createRepoVariable, }); } diff --git a/plugins/scaffolder-backend/migrations/20240203232000_state.js b/plugins/scaffolder-backend/migrations/20240203232000_state.js new file mode 100644 index 0000000000..9ffea1716e --- /dev/null +++ b/plugins/scaffolder-backend/migrations/20240203232000_state.js @@ -0,0 +1,35 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('tasks', table => { + table.text('state').nullable().comment('A state of the checkpoints'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('tasks', table => { + table.dropColumn('state'); + }); +}; diff --git a/plugins/scaffolder-backend/src/index.ts b/plugins/scaffolder-backend/src/index.ts index 649a5df233..d59499739e 100644 --- a/plugins/scaffolder-backend/src/index.ts +++ b/plugins/scaffolder-backend/src/index.ts @@ -23,5 +23,6 @@ export * from './scaffolder'; export * from './service/router'; export * from './lib'; +export { defineCheckpoint } from './util/defineCheckpoint'; export * from './deprecated'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 766db4646e..e6083a469f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -30,6 +30,7 @@ import { TaskStoreCreateTaskResult, TaskStoreShutDownTaskOptions, TaskStoreRecoverTaskOptions, + TaskStoreStateOptions, } from './types'; import { SerializedTaskEvent, @@ -52,6 +53,7 @@ export type RawDbTaskRow = { id: string; spec: string; status: TaskStatus; + state?: string; last_heartbeat_at?: string; created_at: string; created_by: string | null; @@ -394,6 +396,17 @@ export class DatabaseTaskStore implements TaskStore { }); } + async saveCheckpoint?(options: TaskStoreStateOptions): Promise { + if (options.state) { + const serializedState = JSON.stringify(options.state); + await this.db('tasks') + .where({ id: options.taskId }) + .update({ + state: serializedState, + }); + } + } + async listEvents( options: TaskStoreListEventsOptions, ): Promise<{ events: SerializedTaskEvent[] }> { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index ce1ad71c33..190c880280 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -349,6 +349,14 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { logger: taskLogger, logStream: streamLogger, workspacePath, + async checkpoint( + key: string, + fn: () => Promise, + ) { + const value = await fn(); + task.updateCheckpoint?.(key, value); + return value; + }, createTemporaryDirectory: async () => { const tmpDir = await fs.mkdtemp( `${workspacePath}_step-${step.id}-`, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 8b49f492e8..8762253436 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -16,7 +16,7 @@ import { Config } from '@backstage/config'; import { TaskSpec } from '@backstage/plugin-scaffolder-common'; -import { TaskSecrets } from '@backstage/plugin-scaffolder-node'; +import { TaskSecrets, TaskState } from '@backstage/plugin-scaffolder-node'; import { JsonObject, Observable } from '@backstage/types'; import { Logger } from 'winston'; import ObservableImpl from 'zen-observable'; @@ -91,6 +91,14 @@ export class TaskManager implements TaskContext { }); } + async updateCheckpoint?(key: string, value: JsonObject): Promise { + this.task.state = { [key]: value }; + await this.storage.saveCheckpoint?.({ + taskId: this.task.taskId, + state: this.task.state, + }); + } + async complete( result: TaskCompletionState, metadata?: JsonObject, @@ -144,6 +152,10 @@ export interface CurrentClaimedTask { * The secrets that are stored with the task. */ secrets?: TaskSecrets; + /** + * The state of checkpoints of the task. + */ + state?: TaskState; /** * The creator of the task. */ diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index c5783ccb49..b6225d46a9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -16,7 +16,7 @@ import { JsonValue, JsonObject, HumanDuration } from '@backstage/types'; import { TaskSpec, TaskStep } from '@backstage/plugin-scaffolder-common'; -import { TaskSecrets } from '@backstage/plugin-scaffolder-node'; +import { TaskSecrets, TaskState } from '@backstage/plugin-scaffolder-node'; import { TemplateAction, TaskStatus as _TaskStatus, @@ -113,6 +113,11 @@ export type TaskStoreEmitOptions = { body: TBody; }; +export type TaskStoreStateOptions = { + taskId: string; + state?: TaskState; +}; + /** * TaskStoreListEventsOptions * @@ -194,6 +199,8 @@ export interface TaskStore { emitLogEvent(options: TaskStoreEmitOptions): Promise; + saveCheckpoint?(options: TaskStoreStateOptions): Promise; + listEvents( options: TaskStoreListEventsOptions, ): Promise<{ events: SerializedTaskEvent[] }>; diff --git a/plugins/scaffolder-backend/src/util/defineCheckpoint.ts b/plugins/scaffolder-backend/src/util/defineCheckpoint.ts new file mode 100644 index 0000000000..6656005684 --- /dev/null +++ b/plugins/scaffolder-backend/src/util/defineCheckpoint.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonObject } from '@backstage/types'; + +export type DefineCheckpointProps = { + checkpoint?: (key: string, fn: () => Promise) => Promise; + key: string; + fn: () => Promise; +}; + +export const defineCheckpoint = async ( + props: DefineCheckpointProps, +): Promise => { + const { checkpoint, fn, key } = props; + return checkpoint + ? checkpoint?.(key, async () => { + return await fn(); + }) + : fn(); +}; diff --git a/plugins/scaffolder-node/src/actions/types.ts b/plugins/scaffolder-node/src/actions/types.ts index 7e8c2b4f5f..a7f44c4730 100644 --- a/plugins/scaffolder-node/src/actions/types.ts +++ b/plugins/scaffolder-node/src/actions/types.ts @@ -35,6 +35,10 @@ export type ActionContext< secrets?: TaskSecrets; workspacePath: string; input: TActionInput; + checkpoint?( + key: string, + fn: () => Promise, + ): Promise; output( name: keyof TActionOutput, value: TActionOutput[keyof TActionOutput], diff --git a/plugins/scaffolder-node/src/tasks/index.ts b/plugins/scaffolder-node/src/tasks/index.ts index 930de95237..99638e48af 100644 --- a/plugins/scaffolder-node/src/tasks/index.ts +++ b/plugins/scaffolder-node/src/tasks/index.ts @@ -24,5 +24,6 @@ export type { TaskCompletionState, TaskContext, TaskEventType, + TaskState, TaskStatus, } from './types'; diff --git a/plugins/scaffolder-node/src/tasks/types.ts b/plugins/scaffolder-node/src/tasks/types.ts index 7cdc044bdf..7136211671 100644 --- a/plugins/scaffolder-node/src/tasks/types.ts +++ b/plugins/scaffolder-node/src/tasks/types.ts @@ -26,6 +26,13 @@ export type TaskSecrets = Record & { backstageToken?: string; }; +/** + * TaskState + * + * @public + */ +export type TaskState = Record; + /** * The status of each step of the Task * @@ -110,6 +117,7 @@ export interface TaskContext { cancelSignal: AbortSignal; spec: TaskSpec; secrets?: TaskSecrets; + state?: TaskState; createdBy?: string; done: boolean; isDryRun?: boolean; @@ -118,6 +126,8 @@ export interface TaskContext { emitLog(message: string, logMetadata?: JsonObject): Promise; + updateCheckpoint?(key: string, value: JsonObject): Promise; + getWorkspaceName(): Promise; } diff --git a/yarn.lock b/yarn.lock index 8c0ddcabc2..4a18d8c628 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8354,7 +8354,9 @@ __metadata: "@backstage/config": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" + "@backstage/plugin-scaffolder-backend": "workspace:^" "@backstage/plugin-scaffolder-node": "workspace:^" + "@backstage/types": "workspace:^" "@octokit/webhooks": ^10.0.0 "@types/libsodium-wrappers": ^0.7.10 fs-extra: 10.1.0