From e4b50ab39bce59b285bc93838e3946b2d18503c3 Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Tue, 30 Apr 2024 21:10:30 +0200 Subject: [PATCH] Scaffolder workspace serialization Signed-off-by: bnechyporenko --- .changeset/healthy-dots-ring.md | 6 + plugins/scaffolder-backend/api-report.md | 22 ++++ .../migrations/20240401213200_workspace.js | 35 ++++++ plugins/scaffolder-backend/package.json | 2 + .../src/scaffolder/tasks/DatabaseTaskStore.ts | 26 ++++ .../tasks/NunjucksWorkflowRunner.ts | 13 +- .../src/scaffolder/tasks/StorageTaskBroker.ts | 16 +++ .../src/scaffolder/tasks/serializer.test.ts | 111 ++++++++++++++++++ .../src/scaffolder/tasks/serializer.ts | 39 ++++++ .../src/scaffolder/tasks/types.ts | 13 ++ plugins/scaffolder-node/api-report.md | 4 + plugins/scaffolder-node/src/tasks/types.ts | 7 ++ yarn.lock | 2 + 13 files changed, 292 insertions(+), 4 deletions(-) create mode 100644 .changeset/healthy-dots-ring.md create mode 100644 plugins/scaffolder-backend/migrations/20240401213200_workspace.js create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts diff --git a/.changeset/healthy-dots-ring.md b/.changeset/healthy-dots-ring.md new file mode 100644 index 0000000000..005e7e6616 --- /dev/null +++ b/.changeset/healthy-dots-ring.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +'@backstage/plugin-scaffolder-node': patch +--- + +Scaffolder workspace serialization diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 6af1ac04a6..b9bf44b34f 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -3,6 +3,8 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +/// + import { ActionContext as ActionContext_2 } from '@backstage/plugin-scaffolder-node'; import { AuthService } from '@backstage/backend-plugin-api'; import * as azure from '@backstage/plugin-scaffolder-backend-module-azure'; @@ -370,6 +372,8 @@ export interface CurrentClaimedTask { spec: TaskSpec; state?: JsonObject; taskId: string; + // (undocumented) + workspace?: Promise; } // @public @@ -414,6 +418,8 @@ export class DatabaseTaskStore implements TaskStore { | undefined >; // (undocumented) + getWorkspace?(options: { taskId: string }): Promise; + // (undocumented) heartbeatTask(taskId: string): Promise; // (undocumented) list(options: { createdBy?: string }): Promise<{ @@ -437,6 +443,8 @@ export class DatabaseTaskStore implements TaskStore { // (undocumented) saveTaskState(options: { taskId: string; state?: JsonObject }): Promise; // (undocumented) + serializeWorkspace(options: { path: string; taskId: string }): Promise; + // (undocumented) shutdownTask(options: TaskStoreShutDownTaskOptions): Promise; } @@ -554,10 +562,14 @@ export class TaskManager implements TaskContext_2 { | undefined >; // (undocumented) + getWorkspace(options: { taskId: string }): Promise; + // (undocumented) getWorkspaceName(): Promise; // (undocumented) get secrets(): TaskSecrets_2 | undefined; // (undocumented) + serializeWorkspace?(options: { path: string }): Promise; + // (undocumented) get spec(): TaskSpecV1beta3; // (undocumented) updateCheckpoint?( @@ -609,6 +621,8 @@ export interface TaskStore { | undefined >; // (undocumented) + getWorkspace?(options: { taskId: string }): Promise; + // (undocumented) heartbeatTask(taskId: string): Promise; // (undocumented) list?(options: { createdBy?: string }): Promise<{ @@ -634,6 +648,14 @@ export interface TaskStore { state?: JsonObject; }): Promise; // (undocumented) + serializeWorkspace?({ + path, + taskId, + }: { + path: string; + taskId: string; + }): Promise; + // (undocumented) shutdownTask?(options: TaskStoreShutDownTaskOptions): Promise; } diff --git a/plugins/scaffolder-backend/migrations/20240401213200_workspace.js b/plugins/scaffolder-backend/migrations/20240401213200_workspace.js new file mode 100644 index 0000000000..175de54cd9 --- /dev/null +++ b/plugins/scaffolder-backend/migrations/20240401213200_workspace.js @@ -0,0 +1,35 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('tasks', table => { + table.binary('workspace').nullable().comment('A snapshot of the workspace'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('tasks', table => { + table.dropColumn('workspace'); + }); +}; diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index d107d578cb..aef07c731f 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -78,6 +78,7 @@ "@backstage/types": "workspace:^", "@types/express": "^4.17.6", "@types/luxon": "^3.0.0", + "concat-stream": "^2.0.0", "express": "^4.17.1", "express-promise-router": "^4.1.0", "fs-extra": "^11.2.0", @@ -93,6 +94,7 @@ "p-limit": "^3.1.0", "p-queue": "^6.6.2", "prom-client": "^15.0.0", + "tar": "^6.1.12", "uuid": "^9.0.0", "winston": "^3.2.1", "winston-transport": "^4.7.0", diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index ac391ea2b6..7c6606a1a2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -42,6 +42,7 @@ import { DateTime, Duration } from 'luxon'; import { TaskRecovery, TaskSpec } from '@backstage/plugin-scaffolder-common'; import { trimEventsTillLastRecovery } from './taskRecoveryHelper'; import { intervalFromNowTill } from './dbUtil'; +import { restoreWorkspace, serializeWorkspace } from './serializer'; const migrationsDir = resolvePackagePath( '@backstage/plugin-scaffolder-backend', @@ -57,6 +58,7 @@ export type RawDbTaskRow = { created_at: string; created_by: string | null; secrets?: string | null; + workspace?: Buffer; }; export type RawDbTaskEventRow = { @@ -509,6 +511,30 @@ export class DatabaseTaskStore implements TaskStore { }); } + async rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise { + const [result] = await this.db('tasks') + .where({ id: options.taskId }) + .select('workspace'); + + await restoreWorkspace(options.targetPath, result.workspace); + } + + async serializeWorkspace(options: { + path: string; + taskId: string; + }): Promise { + if (options.path) { + await this.db('tasks') + .where({ id: options.taskId }) + .update({ + workspace: await serializeWorkspace(options.path), + }); + } + } + async cancelTask( options: TaskStoreEmitOptions<{ message: string } & JsonObject>, ): Promise { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index 5d24ac7cc7..decb05ad88 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -419,6 +419,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { reason: stringifyError(err), }); throw err; + } finally { + await task.serializeWorkspace?.({ path: workspacePath }); } }, createTemporaryDirectory: async () => { @@ -460,6 +462,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { await taskTrack.markFailed(step, err); await stepTrack.markFailed(); throw err; + } finally { + await task.serializeWorkspace?.({ path: workspacePath }); } } @@ -469,10 +473,9 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { 'Wrong template version executed with the workflow engine', ); } - const workspacePath = path.join( - this.options.workingDirectory, - await task.getWorkspaceName(), - ); + const taskId = await task.getWorkspaceName(); + + const workspacePath = path.join(this.options.workingDirectory, taskId); const { additionalTemplateFilters, additionalTemplateGlobals } = this.options; @@ -486,6 +489,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { }); try { + await task.rehydrateWorkspace?.({ taskId, targetPath: workspacePath }); + const taskTrack = await this.tracker.taskStart(task); await fs.ensureDir(workspacePath); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index e9a408d9d6..51f3afc623 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -99,6 +99,13 @@ export class TaskManager implements TaskContext { return this.task.taskId; } + async rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise { + return this.storage.rehydrateWorkspace?.(options); + } + get done() { return this.isDone; } @@ -144,6 +151,13 @@ export class TaskManager implements TaskContext { }); } + async serializeWorkspace?(options: { path: string }): Promise { + await this.storage.serializeWorkspace?.({ + path: options.path, + taskId: this.task.taskId, + }); + } + async complete( result: TaskCompletionState, metadata?: JsonObject, @@ -219,6 +233,8 @@ export interface CurrentClaimedTask { * The creator of the task. */ createdBy?: string; + + workspace?: Promise; } function defer() { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts new file mode 100644 index 0000000000..bec5398e28 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts @@ -0,0 +1,111 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { serializeWorkspace, restoreWorkspace } from './serializer'; +import { createMockDirectory } from '@backstage/backend-test-utils'; +import fs from 'fs-extra'; + +describe('serializer', () => { + const workspaceDir = createMockDirectory({ + content: { + 'app-config.yaml': ` + app: + title: Example App + sessionKey: + $file: secrets/session-key.txt + escaped: \$\${Escaped} + `, + 'app-config2.yaml': ` + app: + title: Example App 2 + sessionKey: + $file: secrets/session-key.txt + escaped: \$\${Escaped} + `, + 'app-config.development.yaml': ` + app: + sessionKey: development-key + backend: + $include: ./included.yaml + other: + $include: secrets/included.yaml + `, + 'secrets/session-key.txt': 'abc123', + 'secrets/included.yaml': ` + secret: + $file: session-key.txt + `, + 'included.yaml': ` + foo: + bar: token \${MY_SECRET} + `, + 'app-config.substitute.yaml': ` + app: + someConfig: + $include: \${SUBSTITUTE_ME}.yaml + noSubstitute: + $file: \$\${ESCAPE_ME}.txt + `, + 'substituted.yaml': ` + secret: + $file: secrets/\${SUBSTITUTE_ME}.txt + `, + 'secrets/substituted.txt': '123abc', + '${ESCAPE_ME}.txt': 'notSubstituted', + 'empty.yaml': '# just a comment', + }, + }); + + const restoredWorkspaceDir = createMockDirectory(); + + it('should be able to archive and restore the workspace', async () => { + const workspaceBuffer = await serializeWorkspace(workspaceDir.path); + await restoreWorkspace(restoredWorkspaceDir.path, workspaceBuffer); + + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/\$\{ESCAPE_ME\}.txt`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/app-config.development.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/app-config.substitute.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/app-config.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/app-config2.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/empty.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/included.yaml`), + ).toBeTruthy(); + expect(fs.existsSync(`${restoredWorkspaceDir.path}/secrets`)).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/substituted.yaml`), + ).toBeTruthy(); + + expect( + fs.readFileSync(`${restoredWorkspaceDir.path}/substituted.yaml`, 'utf8'), + ).toEqual(` + secret: + $file: secrets/\${SUBSTITUTE_ME}.txt + `); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts new file mode 100644 index 0000000000..69c48fb769 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import tar from 'tar'; +import concatStream from 'concat-stream'; +import { promisify } from 'util'; +import { pipeline as pipelineCb, Readable } from 'stream'; + +const pipeline = promisify(pipelineCb); + +export const serializeWorkspace = async (path: string): Promise => { + return await new Promise(async resolve => { + await pipeline(tar.create({ cwd: path }, ['']), concatStream(resolve)); + }); +}; + +export const restoreWorkspace = async (path: string, buffer?: Buffer) => { + if (buffer) { + await pipeline( + Readable.from(buffer), + tar.extract({ + C: path, + }), + ); + } +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index aa647d8bd6..676b7e8c3a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -211,6 +211,19 @@ export interface TaskStore { ): Promise<{ events: SerializedTaskEvent[] }>; shutdownTask?(options: TaskStoreShutDownTaskOptions): Promise; + + rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise; + + serializeWorkspace?({ + path, + taskId, + }: { + path: string; + taskId: string; + }): Promise; } export type WorkflowResponse = { output: { [key: string]: JsonValue } }; diff --git a/plugins/scaffolder-node/api-report.md b/plugins/scaffolder-node/api-report.md index c757301249..7211dbcb28 100644 --- a/plugins/scaffolder-node/api-report.md +++ b/plugins/scaffolder-node/api-report.md @@ -361,12 +361,16 @@ export interface TaskContext { | undefined >; // (undocumented) + getWorkspace?(options: { taskId: string }): Promise; + // (undocumented) getWorkspaceName(): Promise; // (undocumented) isDryRun?: boolean; // (undocumented) secrets?: TaskSecrets; // (undocumented) + serializeWorkspace?(options: { path: string }): Promise; + // (undocumented) spec: TaskSpec; // (undocumented) updateCheckpoint?( diff --git a/plugins/scaffolder-node/src/tasks/types.ts b/plugins/scaffolder-node/src/tasks/types.ts index aef2c5f360..b2da5b2f0f 100644 --- a/plugins/scaffolder-node/src/tasks/types.ts +++ b/plugins/scaffolder-node/src/tasks/types.ts @@ -141,6 +141,13 @@ export interface TaskContext { }, ): Promise; + serializeWorkspace?(options: { path: string }): Promise; + + rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise; + getWorkspaceName(): Promise; getInitiatorCredentials(): Promise; diff --git a/yarn.lock b/yarn.lock index 3cf5a79175..e9a7947795 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6788,6 +6788,7 @@ __metadata: "@types/nunjucks": ^3.1.4 "@types/supertest": ^2.0.8 "@types/zen-observable": ^0.8.0 + concat-stream: ^2.0.0 esbuild: ^0.20.0 express: ^4.17.1 express-promise-router: ^4.1.0 @@ -6806,6 +6807,7 @@ __metadata: prom-client: ^15.0.0 strip-ansi: ^7.1.0 supertest: ^6.1.3 + tar: ^6.1.12 uuid: ^9.0.0 wait-for-expect: ^3.0.2 winston: ^3.2.1