diff --git a/.changeset/healthy-dots-ring.md b/.changeset/healthy-dots-ring.md new file mode 100644 index 0000000000..005e7e6616 --- /dev/null +++ b/.changeset/healthy-dots-ring.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +'@backstage/plugin-scaffolder-node': patch +--- + +Scaffolder workspace serialization diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 9de99c7a5b..895b55f6e1 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -3,6 +3,8 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +/// + import { ActionContext as ActionContext_2 } from '@backstage/plugin-scaffolder-node'; import { AuthService } from '@backstage/backend-plugin-api'; import * as azure from '@backstage/plugin-scaffolder-backend-module-azure'; @@ -370,6 +372,8 @@ export interface CurrentClaimedTask { spec: TaskSpec; state?: JsonObject; taskId: string; + // (undocumented) + workspace?: Promise; } // @public @@ -385,6 +389,8 @@ export class DatabaseTaskStore implements TaskStore { // (undocumented) claimTask(): Promise; // (undocumented) + cleanWorkspace({ taskId }: { taskId: string }): Promise; + // (undocumented) completeTask(options: { taskId: string; status: TaskStatus_2; @@ -435,8 +441,15 @@ export class DatabaseTaskStore implements TaskStore { ids: string[]; }>; // (undocumented) + rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise; + // (undocumented) saveTaskState(options: { taskId: string; state?: JsonObject }): Promise; // (undocumented) + serializeWorkspace(options: { path: string; taskId: string }): Promise; + // (undocumented) shutdownTask(options: TaskStoreShutDownTaskOptions): Promise; } @@ -529,6 +542,8 @@ export class TaskManager implements TaskContext_2 { // (undocumented) get cancelSignal(): AbortSignal; // (undocumented) + cleanWorkspace?(): Promise; + // (undocumented) complete(result: TaskCompletionState_2, metadata?: JsonObject): Promise; // (undocumented) static create( @@ -537,6 +552,7 @@ export class TaskManager implements TaskContext_2 { abortSignal: AbortSignal, logger: Logger, auth?: AuthService, + config?: Config, ): TaskManager; // (undocumented) get createdBy(): string | undefined; @@ -556,8 +572,15 @@ export class TaskManager implements TaskContext_2 { // (undocumented) getWorkspaceName(): Promise; // (undocumented) + rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise; + // (undocumented) get secrets(): TaskSecrets_2 | undefined; // (undocumented) + serializeWorkspace?(options: { path: string }): Promise; + // (undocumented) get spec(): TaskSpecV1beta3; // (undocumented) updateCheckpoint?( @@ -588,6 +611,8 @@ export interface TaskStore { // (undocumented) claimTask(): Promise; // (undocumented) + cleanWorkspace?({ taskId }: { taskId: string }): Promise; + // (undocumented) completeTask(options: { taskId: string; status: TaskStatus; @@ -629,11 +654,24 @@ export interface TaskStore { ids: string[]; }>; // (undocumented) + rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise; + // (undocumented) saveTaskState?(options: { taskId: string; state?: JsonObject; }): Promise; // (undocumented) + serializeWorkspace?({ + path, + taskId, + }: { + path: string; + taskId: string; + }): Promise; + // (undocumented) shutdownTask?(options: TaskStoreShutDownTaskOptions): Promise; } diff --git a/plugins/scaffolder-backend/config.d.ts b/plugins/scaffolder-backend/config.d.ts index 477defe026..754eedb55b 100644 --- a/plugins/scaffolder-backend/config.d.ts +++ b/plugins/scaffolder-backend/config.d.ts @@ -47,6 +47,11 @@ export interface Config { */ EXPERIMENTAL_recoverTasks?: boolean; + /** + * Sets the serialization of the workspace to have an ability to rerun the failed task. + */ + EXPERIMENTAL_workspaceSerialization?: boolean; + /** * Every task which is in progress state and having a last heartbeat longer than a specified timeout is going to * be attempted to recover. diff --git a/plugins/scaffolder-backend/migrations/20240401213200_workspace.js b/plugins/scaffolder-backend/migrations/20240401213200_workspace.js new file mode 100644 index 0000000000..175de54cd9 --- /dev/null +++ b/plugins/scaffolder-backend/migrations/20240401213200_workspace.js @@ -0,0 +1,35 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('tasks', table => { + table.binary('workspace').nullable().comment('A snapshot of the workspace'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('tasks', table => { + table.dropColumn('workspace'); + }); +}; diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 338dc290e1..3492901386 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -78,6 +78,7 @@ "@backstage/types": "workspace:^", "@types/express": "^4.17.6", "@types/luxon": "^3.0.0", + "concat-stream": "^2.0.0", "express": "^4.17.1", "express-promise-router": "^4.1.0", "fs-extra": "^11.2.0", @@ -93,6 +94,7 @@ "p-limit": "^3.1.0", "p-queue": "^6.6.2", "prom-client": "^15.0.0", + "tar": "^6.1.12", "uuid": "^9.0.0", "winston": "^3.2.1", "winston-transport": "^4.7.0", diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.test.ts index ab80dcff04..f4fae0715b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.test.ts @@ -19,6 +19,8 @@ import { ConfigReader } from '@backstage/config'; import { DatabaseTaskStore } from './DatabaseTaskStore'; import { TaskSpec } from '@backstage/plugin-scaffolder-common'; import { ConflictError } from '@backstage/errors'; +import { createMockDirectory } from '@backstage/backend-test-utils'; +import fs from 'fs-extra'; const createStore = async () => { const manager = DatabaseManager.fromConfig( @@ -37,6 +39,18 @@ const createStore = async () => { return { store, manager }; }; +const workspaceDir = createMockDirectory({ + content: { + 'app-config.yaml': ` + app: + title: Example App + sessionKey: + $file: secrets/session-key.txt + escaped: \$\${Escaped} + `, + }, +}); + describe('DatabaseTaskStore', () => { it('should create the database store and run migration', async () => { const { store, manager } = await createStore(); @@ -259,4 +273,22 @@ describe('DatabaseTaskStore', () => { }, }); }); + + it('serialize and restore the workspace', async () => { + const { store } = await createStore(); + const { taskId } = await store.createTask({ + spec: {} as TaskSpec, + createdBy: 'me', + }); + + await store.serializeWorkspace({ path: workspaceDir.path, taskId }); + expect(fs.existsSync(`${workspaceDir.path}/app-config.yaml`)).toBeTruthy(); + + fs.removeSync(workspaceDir.path); + expect(fs.existsSync(`${workspaceDir.path}/app-config.yaml`)).toBeFalsy(); + + fs.mkdirSync(workspaceDir.path); + await store.rehydrateWorkspace({ targetPath: workspaceDir.path, taskId }); + expect(fs.existsSync(`${workspaceDir.path}/app-config.yaml`)).toBeTruthy(); + }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index ac391ea2b6..f6f00b0441 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -42,6 +42,7 @@ import { DateTime, Duration } from 'luxon'; import { TaskRecovery, TaskSpec } from '@backstage/plugin-scaffolder-common'; import { trimEventsTillLastRecovery } from './taskRecoveryHelper'; import { intervalFromNowTill } from './dbUtil'; +import { restoreWorkspace, serializeWorkspace } from './serializer'; const migrationsDir = resolvePackagePath( '@backstage/plugin-scaffolder-backend', @@ -57,6 +58,7 @@ export type RawDbTaskRow = { created_at: string; created_by: string | null; secrets?: string | null; + workspace?: Buffer; }; export type RawDbTaskEventRow = { @@ -509,6 +511,36 @@ export class DatabaseTaskStore implements TaskStore { }); } + async rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise { + const [result] = await this.db('tasks') + .where({ id: options.taskId }) + .select('workspace'); + + await restoreWorkspace(options.targetPath, result.workspace); + } + + async cleanWorkspace({ taskId }: { taskId: string }): Promise { + await this.db('tasks').where({ id: taskId }).update({ + workspace: undefined, + }); + } + + async serializeWorkspace(options: { + path: string; + taskId: string; + }): Promise { + if (options.path) { + await this.db('tasks') + .where({ id: options.taskId }) + .update({ + workspace: await serializeWorkspace(options.path), + }); + } + } + async cancelTask( options: TaskStoreEmitOptions<{ message: string } & JsonObject>, ): Promise { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index 5d24ac7cc7..77097e97ea 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -419,6 +419,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { reason: stringifyError(err), }); throw err; + } finally { + await task.serializeWorkspace?.({ path: workspacePath }); } }, createTemporaryDirectory: async () => { @@ -455,11 +457,14 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { throw new Error(`Step ${step.name} has been cancelled.`); } + await task.cleanWorkspace?.(); await stepTrack.markSuccessful(); } catch (err) { await taskTrack.markFailed(step, err); await stepTrack.markFailed(); throw err; + } finally { + await task.serializeWorkspace?.({ path: workspacePath }); } } @@ -469,10 +474,9 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { 'Wrong template version executed with the workflow engine', ); } - const workspacePath = path.join( - this.options.workingDirectory, - await task.getWorkspaceName(), - ); + const taskId = await task.getWorkspaceName(); + + const workspacePath = path.join(this.options.workingDirectory, taskId); const { additionalTemplateFilters, additionalTemplateGlobals } = this.options; @@ -486,6 +490,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { }); try { + await task.rehydrateWorkspace?.({ taskId, targetPath: workspacePath }); + const taskTrack = await this.tracker.taskStart(task); await fs.ensureDir(workspacePath); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index e9a408d9d6..03a2833938 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -64,8 +64,16 @@ export class TaskManager implements TaskContext { abortSignal: AbortSignal, logger: Logger, auth?: AuthService, + config?: Config, ) { - const agent = new TaskManager(task, storage, abortSignal, logger, auth); + const agent = new TaskManager( + task, + storage, + abortSignal, + logger, + auth, + config, + ); agent.startTimeout(); return agent; } @@ -77,6 +85,7 @@ export class TaskManager implements TaskContext { private readonly signal: AbortSignal, private readonly logger: Logger, private readonly auth?: AuthService, + private readonly config?: Config, ) {} get spec() { @@ -99,6 +108,15 @@ export class TaskManager implements TaskContext { return this.task.taskId; } + async rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise { + if (this.isWorkspaceSerializationEnabled()) { + this.storage.rehydrateWorkspace?.(options); + } + } + get done() { return this.isDone; } @@ -144,6 +162,21 @@ export class TaskManager implements TaskContext { }); } + async serializeWorkspace?(options: { path: string }): Promise { + if (this.isWorkspaceSerializationEnabled()) { + await this.storage.serializeWorkspace?.({ + path: options.path, + taskId: this.task.taskId, + }); + } + } + + async cleanWorkspace?(): Promise { + if (this.isWorkspaceSerializationEnabled()) { + await this.storage.cleanWorkspace?.({ taskId: this.task.taskId }); + } + } + async complete( result: TaskCompletionState, metadata?: JsonObject, @@ -178,6 +211,14 @@ export class TaskManager implements TaskContext { }, 1000); } + private isWorkspaceSerializationEnabled(): boolean { + return ( + this.config?.getOptionalBoolean( + 'scaffolder.EXPERIMENTAL_workspaceSerialization', + ) ?? false + ); + } + async getInitiatorCredentials(): Promise { const secrets = this.task.secrets as InternalTaskSecrets; @@ -219,6 +260,8 @@ export interface CurrentClaimedTask { * The creator of the task. */ createdBy?: string; + + workspace?: Promise; } function defer() { @@ -322,6 +365,7 @@ export class StorageTaskBroker implements TaskBroker { abortController.signal, this.logger, this.auth, + this.config, ); } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts new file mode 100644 index 0000000000..bec5398e28 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts @@ -0,0 +1,111 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { serializeWorkspace, restoreWorkspace } from './serializer'; +import { createMockDirectory } from '@backstage/backend-test-utils'; +import fs from 'fs-extra'; + +describe('serializer', () => { + const workspaceDir = createMockDirectory({ + content: { + 'app-config.yaml': ` + app: + title: Example App + sessionKey: + $file: secrets/session-key.txt + escaped: \$\${Escaped} + `, + 'app-config2.yaml': ` + app: + title: Example App 2 + sessionKey: + $file: secrets/session-key.txt + escaped: \$\${Escaped} + `, + 'app-config.development.yaml': ` + app: + sessionKey: development-key + backend: + $include: ./included.yaml + other: + $include: secrets/included.yaml + `, + 'secrets/session-key.txt': 'abc123', + 'secrets/included.yaml': ` + secret: + $file: session-key.txt + `, + 'included.yaml': ` + foo: + bar: token \${MY_SECRET} + `, + 'app-config.substitute.yaml': ` + app: + someConfig: + $include: \${SUBSTITUTE_ME}.yaml + noSubstitute: + $file: \$\${ESCAPE_ME}.txt + `, + 'substituted.yaml': ` + secret: + $file: secrets/\${SUBSTITUTE_ME}.txt + `, + 'secrets/substituted.txt': '123abc', + '${ESCAPE_ME}.txt': 'notSubstituted', + 'empty.yaml': '# just a comment', + }, + }); + + const restoredWorkspaceDir = createMockDirectory(); + + it('should be able to archive and restore the workspace', async () => { + const workspaceBuffer = await serializeWorkspace(workspaceDir.path); + await restoreWorkspace(restoredWorkspaceDir.path, workspaceBuffer); + + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/\$\{ESCAPE_ME\}.txt`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/app-config.development.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/app-config.substitute.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/app-config.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/app-config2.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/empty.yaml`), + ).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/included.yaml`), + ).toBeTruthy(); + expect(fs.existsSync(`${restoredWorkspaceDir.path}/secrets`)).toBeTruthy(); + expect( + fs.existsSync(`${restoredWorkspaceDir.path}/substituted.yaml`), + ).toBeTruthy(); + + expect( + fs.readFileSync(`${restoredWorkspaceDir.path}/substituted.yaml`, 'utf8'), + ).toEqual(` + secret: + $file: secrets/\${SUBSTITUTE_ME}.txt + `); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts new file mode 100644 index 0000000000..69c48fb769 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import tar from 'tar'; +import concatStream from 'concat-stream'; +import { promisify } from 'util'; +import { pipeline as pipelineCb, Readable } from 'stream'; + +const pipeline = promisify(pipelineCb); + +export const serializeWorkspace = async (path: string): Promise => { + return await new Promise(async resolve => { + await pipeline(tar.create({ cwd: path }, ['']), concatStream(resolve)); + }); +}; + +export const restoreWorkspace = async (path: string, buffer?: Buffer) => { + if (buffer) { + await pipeline( + Readable.from(buffer), + tar.extract({ + C: path, + }), + ); + } +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index aa647d8bd6..0e0ebd706b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -211,6 +211,21 @@ export interface TaskStore { ): Promise<{ events: SerializedTaskEvent[] }>; shutdownTask?(options: TaskStoreShutDownTaskOptions): Promise; + + rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise; + + cleanWorkspace?({ taskId }: { taskId: string }): Promise; + + serializeWorkspace?({ + path, + taskId, + }: { + path: string; + taskId: string; + }): Promise; } export type WorkflowResponse = { output: { [key: string]: JsonValue } }; diff --git a/plugins/scaffolder-node/api-report.md b/plugins/scaffolder-node/api-report.md index c757301249..66e8ccd678 100644 --- a/plugins/scaffolder-node/api-report.md +++ b/plugins/scaffolder-node/api-report.md @@ -344,6 +344,8 @@ export interface TaskContext { // (undocumented) cancelSignal: AbortSignal; // (undocumented) + cleanWorkspace?(): Promise; + // (undocumented) complete(result: TaskCompletionState, metadata?: JsonObject): Promise; // (undocumented) createdBy?: string; @@ -365,8 +367,15 @@ export interface TaskContext { // (undocumented) isDryRun?: boolean; // (undocumented) + rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise; + // (undocumented) secrets?: TaskSecrets; // (undocumented) + serializeWorkspace?(options: { path: string }): Promise; + // (undocumented) spec: TaskSpec; // (undocumented) updateCheckpoint?( diff --git a/plugins/scaffolder-node/src/tasks/types.ts b/plugins/scaffolder-node/src/tasks/types.ts index aef2c5f360..a2adb97b9c 100644 --- a/plugins/scaffolder-node/src/tasks/types.ts +++ b/plugins/scaffolder-node/src/tasks/types.ts @@ -141,6 +141,15 @@ export interface TaskContext { }, ): Promise; + serializeWorkspace?(options: { path: string }): Promise; + + cleanWorkspace?(): Promise; + + rehydrateWorkspace?(options: { + taskId: string; + targetPath: string; + }): Promise; + getWorkspaceName(): Promise; getInitiatorCredentials(): Promise; diff --git a/yarn.lock b/yarn.lock index 89d4b0ee45..cbdf845ac2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6803,6 +6803,7 @@ __metadata: "@types/nunjucks": ^3.1.4 "@types/supertest": ^2.0.8 "@types/zen-observable": ^0.8.0 + concat-stream: ^2.0.0 esbuild: ^0.20.0 express: ^4.17.1 express-promise-router: ^4.1.0 @@ -6821,6 +6822,7 @@ __metadata: prom-client: ^15.0.0 strip-ansi: ^7.1.0 supertest: ^6.1.3 + tar: ^6.1.12 uuid: ^9.0.0 wait-for-expect: ^3.0.2 winston: ^3.2.1