From e4b50ab39bce59b285bc93838e3946b2d18503c3 Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 30 Apr 2024 21:10:30 +0200 Subject: [PATCH 1/6] Scaffolder workspace serialization Signed-off-by: bnechyporenko --- .changeset/healthy-dots-ring.md | 6 + plugins/scaffolder-backend/api-report.md | 22 ++++ .../migrations/20240401213200_workspace.js | 35 ++++++ plugins/scaffolder-backend/package.json | 2 + .../src/scaffolder/tasks/DatabaseTaskStore.ts | 26 ++++ .../tasks/NunjucksWorkflowRunner.ts | 13 +- .../src/scaffolder/tasks/StorageTaskBroker.ts | 16 +++ .../src/scaffolder/tasks/serializer.test.ts | 111 ++++++++++++++++++ .../src/scaffolder/tasks/serializer.ts | 39 ++++++ .../src/scaffolder/tasks/types.ts | 13 ++ plugins/scaffolder-node/api-report.md | 4 + plugins/scaffolder-node/src/tasks/types.ts | 7 ++ yarn.lock | 2 + 13 files changed, 292 insertions(+), 4 deletions(-) create mode 100644 .changeset/healthy-dots-ring.md create mode 100644 plugins/scaffolder-backend/migrations/20240401213200_workspace.js create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts diff --git a/.changeset/healthy-dots-ring.md b/.changeset/healthy-dots-ring.md new file mode 100644 index 0000000000..005e7e6616 --- /dev/null +++ b/.changeset/healthy-dots-ring.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +'@backstage/plugin-scaffolder-node': patch +--- + +Scaffolder workspace serialization diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 6af1ac04a6..b9bf44b34f 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -3,6 +3,8 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +/// + import { ActionContext as ActionContext_2 } from '@backstage/plugin-scaffolder-node'; import { AuthService } from '@backstage/backend-plugin-api'; import * as azure from '@backstage/plugin-scaffolder-backend-module-azure'; @@ -370,6 +372,8 @@ export interface CurrentClaimedTask { spec: TaskSpec; state?: JsonObject; taskId: string; + // (undocumented) + workspace?: Promise; } // @public @@ -414,6 +418,8 @@ export class DatabaseTaskStore implements TaskStore { | undefined >; // (undocumented) + getWorkspace?(options: { taskId: string }): Promise; + // (undocumented) heartbeatTask(taskId: string): Promise; // (undocumented) list(options: { createdBy?: string }): Promise<{ @@ -437,6 +443,8 @@ export class DatabaseTaskStore implements TaskStore { // (undocumented) saveTaskState(options: { taskId: string; state?: JsonObject }): Promise; // (undocumented) + serializeWorkspace(options: { path: string; taskId: string }): Promise; + // (undocumented) shutdownTask(options: TaskStoreShutDownTaskOptions): Promise; } @@ -554,10 +562,14 @@ export class TaskManager implements TaskContext_2 { | undefined >; // (undocumented) + getWorkspace(options: { taskId: string }): Promise; + // (undocumented) getWorkspaceName(): Promise; // (undocumented) get secrets(): TaskSecrets_2 | undefined; // (undocumented) + serializeWorkspace?(options: { path: string }): Promise; + // (undocumented) get spec(): TaskSpecV1beta3; // (undocumented) updateCheckpoint?( @@ -609,6 +621,8 @@ export interface TaskStore { | undefined >; // (undocumented) + getWorkspace?(options: { taskId: string }): Promise; + // (undocumented) heartbeatTask(taskId: string): Promise; // (undocumented) list?(options: { createdBy?: string }): Promise<{ @@ -634,6 +648,14 @@ export interface TaskStore { state?: JsonObject; }): Promise; // (undocumented) + serializeWorkspace?({ + path, + taskId, + }: { + path: string; + taskId: string; + }): Promise; + // (undocumented) shutdownTask?(options: TaskStoreShutDownTaskOptions): Promise; } diff --git a/plugins/scaffolder-backend/migrations/20240401213200_workspace.js b/plugins/scaffolder-backend/migrations/20240401213200_workspace.js new file mode 100644 index 0000000000..175de54cd9 --- /dev/null +++ b/plugins/scaffolder-backend/migrations/20240401213200_workspace.js @@ -0,0 +1,35 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('tasks', table => { + table.binary('workspace').nullable().comment('A snapshot of the workspace'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('tasks', table => { + table.dropColumn('workspace'); + }); +}; diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index d107d578cb..aef07c731f 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -78,6 +78,7 @@ "@backstage/types": "workspace:^", "@types/express": "^4.17.6", "@types/luxon": "^3.0.0", + "concat-stream": "^2.0.0", "express": "^4.17.1", "express-promise-router": "^4.1.0", "fs-extra": "^11.2.0", @@ -93,6 +94,7 @@ "p-limit": "^3.1.0", "p-queue": "^6.6.2", "prom-client": "^15.0.0", + "tar": "^6.1.12", "uuid": "^9.0.0", "winston": "^3.2.1", "winston-transport": "^4.7.0", diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index ac391ea2b6..7c6606a1a2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -42,6 +42,7 @@ import { DateTime, Duration } from 'luxon'; import { TaskRecovery, TaskSpec } from '@backstage/plugin-scaffolder-common'; import { trimEventsTillLastRecovery } from './taskRecoveryHelper'; import { intervalFromNowTill } from './dbUtil'; +import { restoreWorkspace, serializeWorkspace } from './serializer'; const migrationsDir = resolvePackagePath( '@backstage/plugin-scaffolder-backend', @@ -57,6 +58,7 @@ export type RawDbTaskRow = { created_at: string; created_by: string | null; secrets?: string | null; + workspace?: Buffer; }; export type RawDbTaskEventRow = { @@ -509,6 +511,30 @@ export class DatabaseTaskStore implements TaskStore { }); } + async rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise { + const [result] = await this.db('tasks') + .where({ id: options.taskId }) + .select('workspace'); + + await restoreWorkspace(options.targetPath, result.workspace); + } + + async serializeWorkspace(options: { + path: string; + taskId: string; + }): Promise { + if (options.path) { + await this.db('tasks') + .where({ id: options.taskId }) + .update({ + workspace: await serializeWorkspace(options.path), + }); + } + } + async cancelTask( options: TaskStoreEmitOptions<{ message: string } & JsonObject>, ): Promise { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index 5d24ac7cc7..decb05ad88 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -419,6 +419,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { reason: stringifyError(err), }); throw err; + } finally { + await task.serializeWorkspace?.({ path: workspacePath }); } }, createTemporaryDirectory: async () => { @@ -460,6 +462,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { await taskTrack.markFailed(step, err); await stepTrack.markFailed(); throw err; + } finally { + await task.serializeWorkspace?.({ path: workspacePath }); } } @@ -469,10 +473,9 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { 'Wrong template version executed with the workflow engine', ); } - const workspacePath = path.join( - this.options.workingDirectory, - await task.getWorkspaceName(), - ); + const taskId = await task.getWorkspaceName(); + + const workspacePath = path.join(this.options.workingDirectory, taskId); const { additionalTemplateFilters, additionalTemplateGlobals } = this.options; @@ -486,6 +489,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { }); try { + await task.rehydrateWorkspace?.({ taskId, targetPath: workspacePath }); + const taskTrack = await this.tracker.taskStart(task); await fs.ensureDir(workspacePath); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index e9a408d9d6..51f3afc623 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -99,6 +99,13 @@ export class TaskManager implements TaskContext { return this.task.taskId; } + async rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise { + return this.storage.rehydrateWorkspace?.(options); + } + get done() { return this.isDone; } @@ -144,6 +151,13 @@ export class TaskManager implements TaskContext { }); } + async serializeWorkspace?(options: { path: string }): Promise { + await this.storage.serializeWorkspace?.({ + path: options.path, + taskId: this.task.taskId, + }); + } + async complete( result: TaskCompletionState, metadata?: JsonObject, @@ -219,6 +233,8 @@ export interface CurrentClaimedTask { * The creator of the task. */ createdBy?: string; + + workspace?: Promise; } function defer() { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts new file mode 100644 index 0000000000..bec5398e28 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts @@ -0,0 +1,111 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { serializeWorkspace, restoreWorkspace } from './serializer'; +import { createMockDirectory } from '@backstage/backend-test-utils'; +import fs from 'fs-extra'; + +describe('serializer', () => { + const workspaceDir = createMockDirectory({ + content: { + 'app-config.yaml': ` + app: + title: Example App + sessionKey: + $file: secrets/session-key.txt + escaped: \$\${Escaped} + `, + 'app-config2.yaml': ` + app: + title: Example App 2 + sessionKey: + $file: secrets/session-key.txt + escaped: \$\${Escaped} + `, + 'app-config.development.yaml': ` + app: + sessionKey: development-key + backend: + $include: ./included.yaml + other: + $include: secrets/included.yaml + `, + 'secrets/session-key.txt': 'abc123', + 'secrets/included.yaml': ` + secret: + $file: session-key.txt + `, + 'included.yaml': ` + foo: + bar: token \${MY_SECRET} + `, + 'app-config.substitute.yaml': ` + app: + someConfig: + $include: \${SUBSTITUTE_ME}.yaml + noSubstitute: + $file: \$\${ESCAPE_ME}.txt + `, + 'substituted.yaml': ` + secret: + $file: secrets/\${SUBSTITUTE_ME}.txt + `, + 'secrets/substituted.txt': '123abc', + '${ESCAPE_ME}.txt': 'notSubstituted', + 'empty.yaml': '# just a comment', + }, + }); + + const restoredWorkspaceDir = createMockDirectory(); + + it('should be able to archive and restore the workspace', async () => { + const workspaceBuffer = await serializeWorkspace(workspaceDir.path); + await restoreWorkspace(restoredWorkspaceDir.path, workspaceBuffer); + + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/\$\{ESCAPE_ME\}.txt`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/app-config.development.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/app-config.substitute.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/app-config.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/app-config2.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/empty.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/included.yaml`), + ).toBeTruthy(); + expect(fs.existsSync(`${restoredWorkspaceDir.path}/secrets`)).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/substituted.yaml`), + ).toBeTruthy(); + + expect( + fs.readFileSync(`${restoredWorkspaceDir.path}/substituted.yaml`, 'utf8'), + ).toEqual(` + secret: + $file: secrets/\${SUBSTITUTE_ME}.txt + `); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts new file mode 100644 index 0000000000..69c48fb769 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import tar from 'tar'; +import concatStream from 'concat-stream'; +import { promisify } from 'util'; +import { pipeline as pipelineCb, Readable } from 'stream'; + +const pipeline = promisify(pipelineCb); + +export const serializeWorkspace = async (path: string): Promise => { + return await new Promise(async resolve => { + await pipeline(tar.create({ cwd: path }, ['']), concatStream(resolve)); + }); +}; + +export const restoreWorkspace = async (path: string, buffer?: Buffer) => { + if (buffer) { + await pipeline( + Readable.from(buffer), + tar.extract({ + C: path, + }), + ); + } +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index aa647d8bd6..676b7e8c3a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -211,6 +211,19 @@ export interface TaskStore { ): Promise<{ events: SerializedTaskEvent[] }>; shutdownTask?(options: TaskStoreShutDownTaskOptions): Promise; + + rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise; + + serializeWorkspace?({ + path, + taskId, + }: { + path: string; + taskId: string; + }): Promise; } export type WorkflowResponse = { output: { [key: string]: JsonValue } }; diff --git a/plugins/scaffolder-node/api-report.md b/plugins/scaffolder-node/api-report.md index c757301249..7211dbcb28 100644 --- a/plugins/scaffolder-node/api-report.md +++ b/plugins/scaffolder-node/api-report.md @@ -361,12 +361,16 @@ export interface TaskContext { | undefined >; // (undocumented) + getWorkspace?(options: { taskId: string }): Promise; + // (undocumented) getWorkspaceName(): Promise; // (undocumented) isDryRun?: boolean; // (undocumented) secrets?: TaskSecrets; // (undocumented) + serializeWorkspace?(options: { path: string }): Promise; + // (undocumented) spec: TaskSpec; // (undocumented) updateCheckpoint?( diff --git a/plugins/scaffolder-node/src/tasks/types.ts b/plugins/scaffolder-node/src/tasks/types.ts index aef2c5f360..b2da5b2f0f 100644 --- a/plugins/scaffolder-node/src/tasks/types.ts +++ b/plugins/scaffolder-node/src/tasks/types.ts @@ -141,6 +141,13 @@ export interface TaskContext { }, ): Promise; + serializeWorkspace?(options: { path: string }): Promise; + + rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise; + getWorkspaceName(): Promise; getInitiatorCredentials(): Promise; diff --git a/yarn.lock b/yarn.lock index 3cf5a79175..e9a7947795 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6788,6 +6788,7 @@ __metadata: "@types/nunjucks": ^3.1.4 "@types/supertest": ^2.0.8 "@types/zen-observable": ^0.8.0 + concat-stream: ^2.0.0 esbuild: ^0.20.0 express: ^4.17.1 express-promise-router: ^4.1.0 @@ -6806,6 +6807,7 @@ __metadata: prom-client: ^15.0.0 strip-ansi: ^7.1.0 supertest: ^6.1.3 + tar: ^6.1.12 uuid: ^9.0.0 wait-for-expect: ^3.0.2 winston: ^3.2.1 From ee538cb044075ea4dc0b4266e8f03285ee13c03b Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 30 Apr 2024 21:13:35 +0200 Subject: [PATCH 2/6] Scaffolder workspace serialization Signed-off-by: bnechyporenko --- plugins/scaffolder-backend/api-report.md | 21 +++++++++++++++------ plugins/scaffolder-node/api-report.md | 7 +++++-- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index b9bf44b34f..38ee521b38 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -418,8 +418,6 @@ export class DatabaseTaskStore implements TaskStore { | undefined >; // (undocumented) - getWorkspace?(options: { taskId: string }): Promise; - // (undocumented) heartbeatTask(taskId: string): Promise; // (undocumented) list(options: { createdBy?: string }): Promise<{ @@ -441,6 +439,11 @@ export class DatabaseTaskStore implements TaskStore { ids: string[]; }>; // (undocumented) + rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise; + // (undocumented) saveTaskState(options: { taskId: string; state?: JsonObject }): Promise; // (undocumented) serializeWorkspace(options: { path: string; taskId: string }): Promise; @@ -562,10 +565,13 @@ export class TaskManager implements TaskContext_2 { | undefined >; // (undocumented) - getWorkspace(options: { taskId: string }): Promise; - // (undocumented) getWorkspaceName(): Promise; // (undocumented) + rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise; + // (undocumented) get secrets(): TaskSecrets_2 | undefined; // (undocumented) serializeWorkspace?(options: { path: string }): Promise; @@ -621,8 +627,6 @@ export interface TaskStore { | undefined >; // (undocumented) - getWorkspace?(options: { taskId: string }): Promise; - // (undocumented) heartbeatTask(taskId: string): Promise; // (undocumented) list?(options: { createdBy?: string }): Promise<{ @@ -643,6 +647,11 @@ export interface TaskStore { ids: string[]; }>; // (undocumented) + rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise; + // (undocumented) saveTaskState?(options: { taskId: string; state?: JsonObject; diff --git a/plugins/scaffolder-node/api-report.md b/plugins/scaffolder-node/api-report.md index 7211dbcb28..a6b91bdccb 100644 --- a/plugins/scaffolder-node/api-report.md +++ b/plugins/scaffolder-node/api-report.md @@ -361,12 +361,15 @@ export interface TaskContext { | undefined >; // (undocumented) - getWorkspace?(options: { taskId: string }): Promise; - // (undocumented) getWorkspaceName(): Promise; // (undocumented) isDryRun?: boolean; // (undocumented) + rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise; + // (undocumented) secrets?: TaskSecrets; // (undocumented) serializeWorkspace?(options: { path: string }): Promise; From 425b403bb212a4b8fd41a0a5dfdffcc32c7e9058 Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 30 Apr 2024 21:28:55 +0200 Subject: [PATCH 3/6] Scaffolder workspace serialization Signed-off-by: bnechyporenko --- .../tasks/DatabaseTaskStore.test.ts | 32 +++++++++++++++++++ .../src/scaffolder/tasks/DatabaseTaskStore.ts | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.test.ts index ab80dcff04..f4fae0715b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.test.ts @@ -19,6 +19,8 @@ import { ConfigReader } from '@backstage/config'; import { DatabaseTaskStore } from './DatabaseTaskStore'; import { TaskSpec } from '@backstage/plugin-scaffolder-common'; import { ConflictError } from '@backstage/errors'; +import { createMockDirectory } from '@backstage/backend-test-utils'; +import fs from 'fs-extra'; const createStore = async () => { const manager = DatabaseManager.fromConfig( @@ -37,6 +39,18 @@ const createStore = async () => { return { store, manager }; }; +const workspaceDir = createMockDirectory({ + content: { + 'app-config.yaml': ` + app: + title: Example App + sessionKey: + $file: secrets/session-key.txt + escaped: \$\${Escaped} + `, + }, +}); + describe('DatabaseTaskStore', () => { it('should create the database store and run migration', async () => { const { store, manager } = await createStore(); @@ -259,4 +273,22 @@ describe('DatabaseTaskStore', () => { }, }); }); + + it('serialize and restore the workspace', async () => { + const { store } = await createStore(); + const { taskId } = await store.createTask({ + spec: {} as TaskSpec, + createdBy: 'me', + }); + + await store.serializeWorkspace({ path: workspaceDir.path, taskId }); + expect(fs.existsSync(`${workspaceDir.path}/app-config.yaml`)).toBeTruthy(); + + fs.removeSync(workspaceDir.path); + expect(fs.existsSync(`${workspaceDir.path}/app-config.yaml`)).toBeFalsy(); + + fs.mkdirSync(workspaceDir.path); + await store.rehydrateWorkspace({ targetPath: workspaceDir.path, taskId }); + expect(fs.existsSync(`${workspaceDir.path}/app-config.yaml`)).toBeTruthy(); + }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 7c6606a1a2..792b74e8d2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -511,7 +511,7 @@ export class DatabaseTaskStore implements TaskStore { }); } - async rehydrateWorkspace?(options: { + async rehydrateWorkspace(options: { taskId: string; targetPath: string; }): Promise { From 8ed3cd6fb715fbf8fd8fa9ea871beb1f058cdbd0 Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Wed, 1 May 2024 07:40:33 +0200 Subject: [PATCH 4/6] Scaffolder workspace serialization Signed-off-by: bnechyporenko --- plugins/scaffolder-backend/api-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 38ee521b38..3ef21f4c98 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -439,7 +439,7 @@ export class DatabaseTaskStore implements TaskStore { ids: string[]; }>; // (undocumented) - rehydrateWorkspace?(options: { + rehydrateWorkspace(options: { taskId: string; targetPath: string; }): Promise; From 21aa820c3c186be6e3d41b59f2e009fea9cbc4fa Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Mon, 6 May 2024 18:23:35 +0200 Subject: [PATCH 5/6] wip Signed-off-by: bnechyporenko --- plugins/scaffolder-backend/api-report.md | 11 ++++- plugins/scaffolder-backend/config.d.ts | 5 +++ .../src/scaffolder/tasks/DatabaseTaskStore.ts | 6 +++ .../tasks/NunjucksWorkflowRunner.ts | 1 + .../src/scaffolder/tasks/StorageTaskBroker.ts | 42 +++++++++++++++---- .../src/scaffolder/tasks/types.ts | 2 + plugins/scaffolder-node/api-report.md | 2 + plugins/scaffolder-node/src/tasks/types.ts | 2 + 8 files changed, 63 insertions(+), 8 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 3ef21f4c98..0c01fd6a9e 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -389,6 +389,8 @@ export class DatabaseTaskStore implements TaskStore { // (undocumented) claimTask(): Promise; // (undocumented) + cleanWorkspace({ taskId }: { taskId: string }): Promise; + // (undocumented) completeTask(options: { taskId: string; status: TaskStatus_2; @@ -540,6 +542,8 @@ export class TaskManager implements TaskContext_2 { // (undocumented) get cancelSignal(): AbortSignal; // (undocumented) + cleanWorkspace?(): Promise; + // (undocumented) complete(result: TaskCompletionState_2, metadata?: JsonObject): Promise; // (undocumented) static create( @@ -548,6 +552,7 @@ export class TaskManager implements TaskContext_2 { abortSignal: AbortSignal, logger: Logger, auth?: AuthService, + config?: Config, ): TaskManager; // (undocumented) get createdBy(): string | undefined; @@ -567,7 +572,9 @@ export class TaskManager implements TaskContext_2 { // (undocumented) getWorkspaceName(): Promise; // (undocumented) - rehydrateWorkspace(options: { + get isWorkspaceSerializationEnabled(): boolean; + // (undocumented) + rehydrateWorkspace?(options: { taskId: string; targetPath: string; }): Promise; @@ -606,6 +613,8 @@ export interface TaskStore { // (undocumented) claimTask(): Promise; // (undocumented) + cleanWorkspace?({ taskId }: { taskId: string }): Promise; + // (undocumented) completeTask(options: { taskId: string; status: TaskStatus; diff --git a/plugins/scaffolder-backend/config.d.ts b/plugins/scaffolder-backend/config.d.ts index 477defe026..754eedb55b 100644 --- a/plugins/scaffolder-backend/config.d.ts +++ b/plugins/scaffolder-backend/config.d.ts @@ -47,6 +47,11 @@ export interface Config { */ EXPERIMENTAL_recoverTasks?: boolean; + /** + * Sets the serialization of the workspace to have an ability to rerun the failed task. + */ + EXPERIMENTAL_workspaceSerialization?: boolean; + /** * Every task which is in progress state and having a last heartbeat longer than a specified timeout is going to * be attempted to recover. diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 792b74e8d2..f6f00b0441 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -522,6 +522,12 @@ export class DatabaseTaskStore implements TaskStore { await restoreWorkspace(options.targetPath, result.workspace); } + async cleanWorkspace({ taskId }: { taskId: string }): Promise { + await this.db('tasks').where({ id: taskId }).update({ + workspace: undefined, + }); + } + async serializeWorkspace(options: { path: string; taskId: string; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index decb05ad88..77097e97ea 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -457,6 +457,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { throw new Error(`Step ${step.name} has been cancelled.`); } + await task.cleanWorkspace?.(); await stepTrack.markSuccessful(); } catch (err) { await taskTrack.markFailed(step, err); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 51f3afc623..9f8f7df241 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -64,8 +64,16 @@ export class TaskManager implements TaskContext { abortSignal: AbortSignal, logger: Logger, auth?: AuthService, + config?: Config, ) { - const agent = new TaskManager(task, storage, abortSignal, logger, auth); + const agent = new TaskManager( + task, + storage, + abortSignal, + logger, + auth, + config, + ); agent.startTimeout(); return agent; } @@ -77,8 +85,17 @@ export class TaskManager implements TaskContext { private readonly signal: AbortSignal, private readonly logger: Logger, private readonly auth?: AuthService, + private readonly config?: Config, ) {} + get isWorkspaceSerializationEnabled(): boolean { + return ( + this.config?.getOptionalBoolean( + 'scaffolder.EXPERIMENTAL_workspaceSerialization', + ) ?? false + ); + } + get spec() { return this.task.spec; } @@ -99,11 +116,13 @@ export class TaskManager implements TaskContext { return this.task.taskId; } - async rehydrateWorkspace(options: { + async rehydrateWorkspace?(options: { taskId: string; targetPath: string; }): Promise { - return this.storage.rehydrateWorkspace?.(options); + if (this.isWorkspaceSerializationEnabled) { + this.storage.rehydrateWorkspace?.(options); + } } get done() { @@ -152,10 +171,18 @@ export class TaskManager implements TaskContext { } async serializeWorkspace?(options: { path: string }): Promise { - await this.storage.serializeWorkspace?.({ - path: options.path, - taskId: this.task.taskId, - }); + if (this.isWorkspaceSerializationEnabled) { + await this.storage.serializeWorkspace?.({ + path: options.path, + taskId: this.task.taskId, + }); + } + } + + async cleanWorkspace?(): Promise { + if (this.isWorkspaceSerializationEnabled) { + await this.storage.cleanWorkspace?.({ taskId: this.task.taskId }); + } } async complete( @@ -338,6 +365,7 @@ export class StorageTaskBroker implements TaskBroker { abortController.signal, this.logger, this.auth, + this.config, ); } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 676b7e8c3a..0e0ebd706b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -217,6 +217,8 @@ export interface TaskStore { targetPath: string; }): Promise; + cleanWorkspace?({ taskId }: { taskId: string }): Promise; + serializeWorkspace?({ path, taskId, diff --git a/plugins/scaffolder-node/api-report.md b/plugins/scaffolder-node/api-report.md index a6b91bdccb..66e8ccd678 100644 --- a/plugins/scaffolder-node/api-report.md +++ b/plugins/scaffolder-node/api-report.md @@ -344,6 +344,8 @@ export interface TaskContext { // (undocumented) cancelSignal: AbortSignal; // (undocumented) + cleanWorkspace?(): Promise; + // (undocumented) complete(result: TaskCompletionState, metadata?: JsonObject): Promise; // (undocumented) createdBy?: string; diff --git a/plugins/scaffolder-node/src/tasks/types.ts b/plugins/scaffolder-node/src/tasks/types.ts index b2da5b2f0f..a2adb97b9c 100644 --- a/plugins/scaffolder-node/src/tasks/types.ts +++ b/plugins/scaffolder-node/src/tasks/types.ts @@ -143,6 +143,8 @@ export interface TaskContext { serializeWorkspace?(options: { path: string }): Promise; + cleanWorkspace?(): Promise; + rehydrateWorkspace?(options: { taskId: string; targetPath: string; From 3d871b99f9ac7668c37bd8e5c84926afd923972d Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 7 May 2024 11:23:14 +0200 Subject: [PATCH 6/6] chore: move to private method over gettter Signed-off-by: blam --- plugins/scaffolder-backend/api-report.md | 2 -- .../src/scaffolder/tasks/StorageTaskBroker.ts | 22 +++++++++---------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 0c01fd6a9e..8caa0ab0d4 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -572,8 +572,6 @@ export class TaskManager implements TaskContext_2 { // (undocumented) getWorkspaceName(): Promise; // (undocumented) - get isWorkspaceSerializationEnabled(): boolean; - // (undocumented) rehydrateWorkspace?(options: { taskId: string; targetPath: string; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 9f8f7df241..03a2833938 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -88,14 +88,6 @@ export class TaskManager implements TaskContext { private readonly config?: Config, ) {} - get isWorkspaceSerializationEnabled(): boolean { - return ( - this.config?.getOptionalBoolean( - 'scaffolder.EXPERIMENTAL_workspaceSerialization', - ) ?? false - ); - } - get spec() { return this.task.spec; } @@ -120,7 +112,7 @@ export class TaskManager implements TaskContext { taskId: string; targetPath: string; }): Promise { - if (this.isWorkspaceSerializationEnabled) { + if (this.isWorkspaceSerializationEnabled()) { this.storage.rehydrateWorkspace?.(options); } } @@ -171,7 +163,7 @@ export class TaskManager implements TaskContext { } async serializeWorkspace?(options: { path: string }): Promise { - if (this.isWorkspaceSerializationEnabled) { + if (this.isWorkspaceSerializationEnabled()) { await this.storage.serializeWorkspace?.({ path: options.path, taskId: this.task.taskId, @@ -180,7 +172,7 @@ export class TaskManager implements TaskContext { } async cleanWorkspace?(): Promise { - if (this.isWorkspaceSerializationEnabled) { + if (this.isWorkspaceSerializationEnabled()) { await this.storage.cleanWorkspace?.({ taskId: this.task.taskId }); } } @@ -219,6 +211,14 @@ export class TaskManager implements TaskContext { }, 1000); } + private isWorkspaceSerializationEnabled(): boolean { + return ( + this.config?.getOptionalBoolean( + 'scaffolder.EXPERIMENTAL_workspaceSerialization', + ) ?? false + ); + } + async getInitiatorCredentials(): Promise { const secrets = this.task.secrets as InternalTaskSecrets;