From 21aa820c3c186be6e3d41b59f2e009fea9cbc4fa Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Mon, 6 May 2024 18:23:35 +0200 Subject: [PATCH] wip Signed-off-by: bnechyporenko --- plugins/scaffolder-backend/api-report.md | 11 ++++- plugins/scaffolder-backend/config.d.ts | 5 +++ .../src/scaffolder/tasks/DatabaseTaskStore.ts | 6 +++ .../tasks/NunjucksWorkflowRunner.ts | 1 + .../src/scaffolder/tasks/StorageTaskBroker.ts | 42 +++++++++++++++---- .../src/scaffolder/tasks/types.ts | 2 + plugins/scaffolder-node/api-report.md | 2 + plugins/scaffolder-node/src/tasks/types.ts | 2 + 8 files changed, 63 insertions(+), 8 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 3ef21f4c98..0c01fd6a9e 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -389,6 +389,8 @@ export class DatabaseTaskStore implements TaskStore { // (undocumented) claimTask(): Promise; // (undocumented) + cleanWorkspace({ taskId }: { taskId: string }): Promise; + // (undocumented) completeTask(options: { taskId: string; status: TaskStatus_2; @@ -540,6 +542,8 @@ export class TaskManager implements TaskContext_2 { // (undocumented) get cancelSignal(): AbortSignal; // (undocumented) + cleanWorkspace?(): Promise; + // (undocumented) complete(result: TaskCompletionState_2, metadata?: JsonObject): Promise; // (undocumented) static create( @@ -548,6 +552,7 @@ export class TaskManager implements TaskContext_2 { abortSignal: AbortSignal, logger: Logger, auth?: AuthService, + config?: Config, ): TaskManager; // (undocumented) get createdBy(): string | undefined; @@ -567,7 +572,9 @@ export class TaskManager implements TaskContext_2 { // (undocumented) getWorkspaceName(): Promise; // (undocumented) - rehydrateWorkspace(options: { + get isWorkspaceSerializationEnabled(): boolean; + // (undocumented) + rehydrateWorkspace?(options: { taskId: string; targetPath: string; }): Promise; @@ -606,6 +613,8 @@ export interface TaskStore { // (undocumented) claimTask(): Promise; // (undocumented) + cleanWorkspace?({ taskId }: { taskId: string }): Promise; + // (undocumented) completeTask(options: { taskId: string; status: TaskStatus; diff --git a/plugins/scaffolder-backend/config.d.ts b/plugins/scaffolder-backend/config.d.ts index 477defe026..754eedb55b 100644 --- a/plugins/scaffolder-backend/config.d.ts +++ b/plugins/scaffolder-backend/config.d.ts @@ -47,6 +47,11 @@ export interface Config { */ EXPERIMENTAL_recoverTasks?: boolean; + /** + * Sets the serialization of the workspace to have an ability to rerun the failed task. + */ + EXPERIMENTAL_workspaceSerialization?: boolean; + /** * Every task which is in progress state and having a last heartbeat longer than a specified timeout is going to * be attempted to recover. diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 792b74e8d2..f6f00b0441 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -522,6 +522,12 @@ export class DatabaseTaskStore implements TaskStore { await restoreWorkspace(options.targetPath, result.workspace); } + async cleanWorkspace({ taskId }: { taskId: string }): Promise { + await this.db('tasks').where({ id: taskId }).update({ + workspace: undefined, + }); + } + async serializeWorkspace(options: { path: string; taskId: string; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index decb05ad88..77097e97ea 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -457,6 +457,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { throw new Error(`Step ${step.name} has been cancelled.`); } + await task.cleanWorkspace?.(); await stepTrack.markSuccessful(); } catch (err) { await taskTrack.markFailed(step, err); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 51f3afc623..9f8f7df241 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -64,8 +64,16 @@ export class TaskManager implements TaskContext { abortSignal: AbortSignal, logger: Logger, auth?: AuthService, + config?: Config, ) { - const agent = new TaskManager(task, storage, abortSignal, logger, auth); + const agent = new TaskManager( + task, + storage, + abortSignal, + logger, + auth, + config, + ); agent.startTimeout(); return agent; } @@ -77,8 +85,17 @@ export class TaskManager implements TaskContext { private readonly signal: AbortSignal, private readonly logger: Logger, private readonly auth?: AuthService, + private readonly config?: Config, ) {} + get isWorkspaceSerializationEnabled(): boolean { + return ( + this.config?.getOptionalBoolean( + 'scaffolder.EXPERIMENTAL_workspaceSerialization', + ) ?? false + ); + } + get spec() { return this.task.spec; } @@ -99,11 +116,13 @@ export class TaskManager implements TaskContext { return this.task.taskId; } - async rehydrateWorkspace(options: { + async rehydrateWorkspace?(options: { taskId: string; targetPath: string; }): Promise { - return this.storage.rehydrateWorkspace?.(options); + if (this.isWorkspaceSerializationEnabled) { + this.storage.rehydrateWorkspace?.(options); + } } get done() { @@ -152,10 +171,18 @@ export class TaskManager implements TaskContext { } async serializeWorkspace?(options: { path: string }): Promise { - await this.storage.serializeWorkspace?.({ - path: options.path, - taskId: this.task.taskId, - }); + if (this.isWorkspaceSerializationEnabled) { + await this.storage.serializeWorkspace?.({ + path: options.path, + taskId: this.task.taskId, + }); + } + } + + async cleanWorkspace?(): Promise { + if (this.isWorkspaceSerializationEnabled) { + await this.storage.cleanWorkspace?.({ taskId: this.task.taskId }); + } } async complete( @@ -338,6 +365,7 @@ export class StorageTaskBroker implements TaskBroker { abortController.signal, this.logger, this.auth, + this.config, ); } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 676b7e8c3a..0e0ebd706b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -217,6 +217,8 @@ export interface TaskStore { targetPath: string; }): Promise; + cleanWorkspace?({ taskId }: { taskId: string }): Promise; + serializeWorkspace?({ path, taskId, diff --git a/plugins/scaffolder-node/api-report.md b/plugins/scaffolder-node/api-report.md index a6b91bdccb..66e8ccd678 100644 --- a/plugins/scaffolder-node/api-report.md +++ b/plugins/scaffolder-node/api-report.md @@ -344,6 +344,8 @@ export interface TaskContext { // (undocumented) cancelSignal: AbortSignal; // (undocumented) + cleanWorkspace?(): Promise; + // (undocumented) complete(result: TaskCompletionState, metadata?: JsonObject): Promise; // (undocumented) createdBy?: string; diff --git a/plugins/scaffolder-node/src/tasks/types.ts b/plugins/scaffolder-node/src/tasks/types.ts index b2da5b2f0f..a2adb97b9c 100644 --- a/plugins/scaffolder-node/src/tasks/types.ts +++ b/plugins/scaffolder-node/src/tasks/types.ts @@ -143,6 +143,8 @@ export interface TaskContext { serializeWorkspace?(options: { path: string }): Promise; + cleanWorkspace?(): Promise; + rehydrateWorkspace?(options: { taskId: string; targetPath: string;