Merge pull request #24570 from acierto/serder-workspace

Scaffolder workspace serialization

.changeset/healthy-dots-ring.md

@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+'@backstage/plugin-scaffolder-node': patch
+---
+
+Scaffolder workspace serialization

plugins/scaffolder-backend/api-report.md

@@ -3,6 +3,8 @@
 > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
 
 ```ts
+/// <reference types="node" />
+
 import { ActionContext as ActionContext_2 } from '@backstage/plugin-scaffolder-node';
 import { AuthService } from '@backstage/backend-plugin-api';
 import * as azure from '@backstage/plugin-scaffolder-backend-module-azure';
@@ -370,6 +372,8 @@ export interface CurrentClaimedTask {
   spec: TaskSpec;
   state?: JsonObject;
   taskId: string;
+  // (undocumented)
+  workspace?: Promise<Buffer>;
 }
 
 // @public
@@ -385,6 +389,8 @@ export class DatabaseTaskStore implements TaskStore {
   // (undocumented)
   claimTask(): Promise<SerializedTask_2 | undefined>;
   // (undocumented)
+  cleanWorkspace({ taskId }: { taskId: string }): Promise<void>;
+  // (undocumented)
   completeTask(options: {
     taskId: string;
     status: TaskStatus_2;
@@ -435,8 +441,15 @@ export class DatabaseTaskStore implements TaskStore {
     ids: string[];
   }>;
   // (undocumented)
+  rehydrateWorkspace(options: {
+    taskId: string;
+    targetPath: string;
+  }): Promise<void>;
+  // (undocumented)
   saveTaskState(options: { taskId: string; state?: JsonObject }): Promise<void>;
   // (undocumented)
+  serializeWorkspace(options: { path: string; taskId: string }): Promise<void>;
+  // (undocumented)
   shutdownTask(options: TaskStoreShutDownTaskOptions): Promise<void>;
 }
 
@@ -529,6 +542,8 @@ export class TaskManager implements TaskContext_2 {
   // (undocumented)
   get cancelSignal(): AbortSignal;
   // (undocumented)
+  cleanWorkspace?(): Promise<void>;
+  // (undocumented)
   complete(result: TaskCompletionState_2, metadata?: JsonObject): Promise<void>;
   // (undocumented)
   static create(
@@ -537,6 +552,7 @@ export class TaskManager implements TaskContext_2 {
     abortSignal: AbortSignal,
     logger: Logger,
     auth?: AuthService,
+    config?: Config,
   ): TaskManager;
   // (undocumented)
   get createdBy(): string | undefined;
@@ -556,8 +572,15 @@ export class TaskManager implements TaskContext_2 {
   // (undocumented)
   getWorkspaceName(): Promise<string>;
   // (undocumented)
+  rehydrateWorkspace?(options: {
+    taskId: string;
+    targetPath: string;
+  }): Promise<void>;
+  // (undocumented)
   get secrets(): TaskSecrets_2 | undefined;
   // (undocumented)
+  serializeWorkspace?(options: { path: string }): Promise<void>;
+  // (undocumented)
   get spec(): TaskSpecV1beta3;
   // (undocumented)
   updateCheckpoint?(
@@ -588,6 +611,8 @@ export interface TaskStore {
   // (undocumented)
   claimTask(): Promise<SerializedTask | undefined>;
   // (undocumented)
+  cleanWorkspace?({ taskId }: { taskId: string }): Promise<void>;
+  // (undocumented)
   completeTask(options: {
     taskId: string;
     status: TaskStatus;
@@ -629,11 +654,24 @@ export interface TaskStore {
     ids: string[];
   }>;
   // (undocumented)
+  rehydrateWorkspace?(options: {
+    taskId: string;
+    targetPath: string;
+  }): Promise<void>;
+  // (undocumented)
   saveTaskState?(options: {
     taskId: string;
     state?: JsonObject;
   }): Promise<void>;
   // (undocumented)
+  serializeWorkspace?({
+    path,
+    taskId,
+  }: {
+    path: string;
+    taskId: string;
+  }): Promise<void>;
+  // (undocumented)
   shutdownTask?(options: TaskStoreShutDownTaskOptions): Promise<void>;
 }
 

plugins/scaffolder-backend/config.d.ts

@@ -47,6 +47,11 @@ export interface Config {
      */
     EXPERIMENTAL_recoverTasks?: boolean;
 
+    /**
+     * Sets the serialization of the workspace to have an ability to rerun the failed task.
+     */
+    EXPERIMENTAL_workspaceSerialization?: boolean;
+
     /**
      * Every task which is in progress state and having a last heartbeat longer than a specified timeout is going to
      * be attempted to recover.

plugins/scaffolder-backend/migrations/20240401213200_workspace.js

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.up = async function up(knex) {
+  await knex.schema.alterTable('tasks', table => {
+    table.binary('workspace').nullable().comment('A snapshot of the workspace');
+  });
+};
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.down = async function down(knex) {
+  await knex.schema.alterTable('tasks', table => {
+    table.dropColumn('workspace');
+  });
+};

plugins/scaffolder-backend/package.json

@@ -78,6 +78,7 @@
     "@backstage/types": "workspace:^",
     "@types/express": "^4.17.6",
     "@types/luxon": "^3.0.0",
+    "concat-stream": "^2.0.0",
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",
     "fs-extra": "^11.2.0",
@@ -93,6 +94,7 @@
     "p-limit": "^3.1.0",
     "p-queue": "^6.6.2",
     "prom-client": "^15.0.0",
+    "tar": "^6.1.12",
     "uuid": "^9.0.0",
     "winston": "^3.2.1",
     "winston-transport": "^4.7.0",

plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.test.ts

@@ -19,6 +19,8 @@ import { ConfigReader } from '@backstage/config';
 import { DatabaseTaskStore } from './DatabaseTaskStore';
 import { TaskSpec } from '@backstage/plugin-scaffolder-common';
 import { ConflictError } from '@backstage/errors';
+import { createMockDirectory } from '@backstage/backend-test-utils';
+import fs from 'fs-extra';
 
 const createStore = async () => {
   const manager = DatabaseManager.fromConfig(
@@ -37,6 +39,18 @@ const createStore = async () => {
   return { store, manager };
 };
 
+const workspaceDir = createMockDirectory({
+  content: {
+    'app-config.yaml': `
+            app:
+              title: Example App
+              sessionKey:
+                $file: secrets/session-key.txt
+              escaped: \$\${Escaped}
+          `,
+  },
+});
+
 describe('DatabaseTaskStore', () => {
   it('should create the database store and run migration', async () => {
     const { store, manager } = await createStore();
@@ -259,4 +273,22 @@ describe('DatabaseTaskStore', () => {
       },
     });
   });
+
+  it('serialize and restore the workspace', async () => {
+    const { store } = await createStore();
+    const { taskId } = await store.createTask({
+      spec: {} as TaskSpec,
+      createdBy: 'me',
+    });
+
+    await store.serializeWorkspace({ path: workspaceDir.path, taskId });
+    expect(fs.existsSync(`${workspaceDir.path}/app-config.yaml`)).toBeTruthy();
+
+    fs.removeSync(workspaceDir.path);
+    expect(fs.existsSync(`${workspaceDir.path}/app-config.yaml`)).toBeFalsy();
+
+    fs.mkdirSync(workspaceDir.path);
+    await store.rehydrateWorkspace({ targetPath: workspaceDir.path, taskId });
+    expect(fs.existsSync(`${workspaceDir.path}/app-config.yaml`)).toBeTruthy();
+  });
 });

plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts

@@ -42,6 +42,7 @@ import { DateTime, Duration } from 'luxon';
 import { TaskRecovery, TaskSpec } from '@backstage/plugin-scaffolder-common';
 import { trimEventsTillLastRecovery } from './taskRecoveryHelper';
 import { intervalFromNowTill } from './dbUtil';
+import { restoreWorkspace, serializeWorkspace } from './serializer';
 
 const migrationsDir = resolvePackagePath(
   '@backstage/plugin-scaffolder-backend',
@@ -57,6 +58,7 @@ export type RawDbTaskRow = {
   created_at: string;
   created_by: string | null;
   secrets?: string | null;
+  workspace?: Buffer;
 };
 
 export type RawDbTaskEventRow = {
@@ -509,6 +511,36 @@ export class DatabaseTaskStore implements TaskStore {
     });
   }
 
+  async rehydrateWorkspace(options: {
+    taskId: string;
+    targetPath: string;
+  }): Promise<void> {
+    const [result] = await this.db<RawDbTaskRow>('tasks')
+      .where({ id: options.taskId })
+      .select('workspace');
+
+    await restoreWorkspace(options.targetPath, result.workspace);
+  }
+
+  async cleanWorkspace({ taskId }: { taskId: string }): Promise<void> {
+    await this.db<RawDbTaskRow>('tasks').where({ id: taskId }).update({
+      workspace: undefined,
+    });
+  }
+
+  async serializeWorkspace(options: {
+    path: string;
+    taskId: string;
+  }): Promise<void> {
+    if (options.path) {
+      await this.db<RawDbTaskRow>('tasks')
+        .where({ id: options.taskId })
+        .update({
+          workspace: await serializeWorkspace(options.path),
+        });
+    }
+  }
+
   async cancelTask(
     options: TaskStoreEmitOptions<{ message: string } & JsonObject>,
   ): Promise<void> {

plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts

@@ -419,6 +419,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
                 reason: stringifyError(err),
               });
               throw err;
+            } finally {
+              await task.serializeWorkspace?.({ path: workspacePath });
             }
           },
           createTemporaryDirectory: async () => {
@@ -455,11 +457,14 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
         throw new Error(`Step ${step.name} has been cancelled.`);
       }
 
+      await task.cleanWorkspace?.();
       await stepTrack.markSuccessful();
     } catch (err) {
       await taskTrack.markFailed(step, err);
       await stepTrack.markFailed();
       throw err;
+    } finally {
+      await task.serializeWorkspace?.({ path: workspacePath });
     }
   }
 
@@ -469,10 +474,9 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
         'Wrong template version executed with the workflow engine',
       );
     }
-    const workspacePath = path.join(
-      this.options.workingDirectory,
-      await task.getWorkspaceName(),
-    );
+    const taskId = await task.getWorkspaceName();
+
+    const workspacePath = path.join(this.options.workingDirectory, taskId);
 
     const { additionalTemplateFilters, additionalTemplateGlobals } =
       this.options;
@@ -486,6 +490,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
     });
 
     try {
+      await task.rehydrateWorkspace?.({ taskId, targetPath: workspacePath });
+
       const taskTrack = await this.tracker.taskStart(task);
       await fs.ensureDir(workspacePath);
 

plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts

@@ -64,8 +64,16 @@ export class TaskManager implements TaskContext {
     abortSignal: AbortSignal,
     logger: Logger,
     auth?: AuthService,
+    config?: Config,
   ) {
-    const agent = new TaskManager(task, storage, abortSignal, logger, auth);
+    const agent = new TaskManager(
+      task,
+      storage,
+      abortSignal,
+      logger,
+      auth,
+      config,
+    );
     agent.startTimeout();
     return agent;
   }
@@ -77,6 +85,7 @@ export class TaskManager implements TaskContext {
     private readonly signal: AbortSignal,
     private readonly logger: Logger,
     private readonly auth?: AuthService,
+    private readonly config?: Config,
   ) {}
 
   get spec() {
@@ -99,6 +108,15 @@ export class TaskManager implements TaskContext {
     return this.task.taskId;
   }
 
+  async rehydrateWorkspace?(options: {
+    taskId: string;
+    targetPath: string;
+  }): Promise<void> {
+    if (this.isWorkspaceSerializationEnabled()) {
+      this.storage.rehydrateWorkspace?.(options);
+    }
+  }
+
   get done() {
     return this.isDone;
   }
@@ -144,6 +162,21 @@ export class TaskManager implements TaskContext {
     });
   }
 
+  async serializeWorkspace?(options: { path: string }): Promise<void> {
+    if (this.isWorkspaceSerializationEnabled()) {
+      await this.storage.serializeWorkspace?.({
+        path: options.path,
+        taskId: this.task.taskId,
+      });
+    }
+  }
+
+  async cleanWorkspace?(): Promise<void> {
+    if (this.isWorkspaceSerializationEnabled()) {
+      await this.storage.cleanWorkspace?.({ taskId: this.task.taskId });
+    }
+  }
+
   async complete(
     result: TaskCompletionState,
     metadata?: JsonObject,
@@ -178,6 +211,14 @@ export class TaskManager implements TaskContext {
     }, 1000);
   }
 
+  private isWorkspaceSerializationEnabled(): boolean {
+    return (
+      this.config?.getOptionalBoolean(
+        'scaffolder.EXPERIMENTAL_workspaceSerialization',
+      ) ?? false
+    );
+  }
+
   async getInitiatorCredentials(): Promise<BackstageCredentials> {
     const secrets = this.task.secrets as InternalTaskSecrets;
 
@@ -219,6 +260,8 @@ export interface CurrentClaimedTask {
    * The creator of the task.
    */
   createdBy?: string;
+
+  workspace?: Promise<Buffer>;
 }
 
 function defer() {
@@ -322,6 +365,7 @@ export class StorageTaskBroker implements TaskBroker {
           abortController.signal,
           this.logger,
           this.auth,
+          this.config,
         );
       }
 

plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { serializeWorkspace, restoreWorkspace } from './serializer';
+import { createMockDirectory } from '@backstage/backend-test-utils';
+import fs from 'fs-extra';
+
+describe('serializer', () => {
+  const workspaceDir = createMockDirectory({
+    content: {
+      'app-config.yaml': `
+            app:
+              title: Example App
+              sessionKey:
+                $file: secrets/session-key.txt
+              escaped: \$\${Escaped}
+          `,
+      'app-config2.yaml': `
+            app:
+              title: Example App 2
+              sessionKey:
+                $file: secrets/session-key.txt
+              escaped: \$\${Escaped}
+          `,
+      'app-config.development.yaml': `
+            app:
+              sessionKey: development-key
+            backend:
+              $include: ./included.yaml
+            other:
+              $include: secrets/included.yaml
+          `,
+      'secrets/session-key.txt': 'abc123',
+      'secrets/included.yaml': `
+            secret:
+              $file: session-key.txt
+          `,
+      'included.yaml': `
+            foo:
+              bar: token \${MY_SECRET}
+          `,
+      'app-config.substitute.yaml': `
+            app:
+              someConfig:
+                $include: \${SUBSTITUTE_ME}.yaml
+              noSubstitute:
+                $file: \$\${ESCAPE_ME}.txt
+          `,
+      'substituted.yaml': `
+            secret:
+              $file: secrets/\${SUBSTITUTE_ME}.txt
+          `,
+      'secrets/substituted.txt': '123abc',
+      '${ESCAPE_ME}.txt': 'notSubstituted',
+      'empty.yaml': '# just a comment',
+    },
+  });
+
+  const restoredWorkspaceDir = createMockDirectory();
+
+  it('should be able to archive and restore the workspace', async () => {
+    const workspaceBuffer = await serializeWorkspace(workspaceDir.path);
+    await restoreWorkspace(restoredWorkspaceDir.path, workspaceBuffer);
+
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/\$\{ESCAPE_ME\}.txt`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/app-config.development.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/app-config.substitute.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/app-config.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/app-config2.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/empty.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/included.yaml`),
+    ).toBeTruthy();
+    expect(fs.existsSync(`${restoredWorkspaceDir.path}/secrets`)).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/substituted.yaml`),
+    ).toBeTruthy();
+
+    expect(
+      fs.readFileSync(`${restoredWorkspaceDir.path}/substituted.yaml`, 'utf8'),
+    ).toEqual(`
+            secret:
+              $file: secrets/\${SUBSTITUTE_ME}.txt
+          `);
+  });
+});

plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import tar from 'tar';
+import concatStream from 'concat-stream';
+import { promisify } from 'util';
+import { pipeline as pipelineCb, Readable } from 'stream';
+
+const pipeline = promisify(pipelineCb);
+
+export const serializeWorkspace = async (path: string): Promise<Buffer> => {
+  return await new Promise<Buffer>(async resolve => {
+    await pipeline(tar.create({ cwd: path }, ['']), concatStream(resolve));
+  });
+};
+
+export const restoreWorkspace = async (path: string, buffer?: Buffer) => {
+  if (buffer) {
+    await pipeline(
+      Readable.from(buffer),
+      tar.extract({
+        C: path,
+      }),
+    );
+  }
+};

plugins/scaffolder-backend/src/scaffolder/tasks/types.ts

@@ -211,6 +211,21 @@ export interface TaskStore {
   ): Promise<{ events: SerializedTaskEvent[] }>;
 
   shutdownTask?(options: TaskStoreShutDownTaskOptions): Promise<void>;
+
+  rehydrateWorkspace?(options: {
+    taskId: string;
+    targetPath: string;
+  }): Promise<void>;
+
+  cleanWorkspace?({ taskId }: { taskId: string }): Promise<void>;
+
+  serializeWorkspace?({
+    path,
+    taskId,
+  }: {
+    path: string;
+    taskId: string;
+  }): Promise<void>;
 }
 
 export type WorkflowResponse = { output: { [key: string]: JsonValue } };

plugins/scaffolder-node/api-report.md

@@ -344,6 +344,8 @@ export interface TaskContext {
   // (undocumented)
   cancelSignal: AbortSignal;
   // (undocumented)
+  cleanWorkspace?(): Promise<void>;
+  // (undocumented)
   complete(result: TaskCompletionState, metadata?: JsonObject): Promise<void>;
   // (undocumented)
   createdBy?: string;
@@ -365,8 +367,15 @@ export interface TaskContext {
   // (undocumented)
   isDryRun?: boolean;
   // (undocumented)
+  rehydrateWorkspace?(options: {
+    taskId: string;
+    targetPath: string;
+  }): Promise<void>;
+  // (undocumented)
   secrets?: TaskSecrets;
   // (undocumented)
+  serializeWorkspace?(options: { path: string }): Promise<void>;
+  // (undocumented)
   spec: TaskSpec;
   // (undocumented)
   updateCheckpoint?(

plugins/scaffolder-node/src/tasks/types.ts

@@ -141,6 +141,15 @@ export interface TaskContext {
         },
   ): Promise<void>;
 
+  serializeWorkspace?(options: { path: string }): Promise<void>;
+
+  cleanWorkspace?(): Promise<void>;
+
+  rehydrateWorkspace?(options: {
+    taskId: string;
+    targetPath: string;
+  }): Promise<void>;
+
   getWorkspaceName(): Promise<string>;
 
   getInitiatorCredentials(): Promise<BackstageCredentials>;

yarn.lock

@@ -6803,6 +6803,7 @@ __metadata:
     "@types/nunjucks": ^3.1.4
     "@types/supertest": ^2.0.8
     "@types/zen-observable": ^0.8.0
+    concat-stream: ^2.0.0
     esbuild: ^0.20.0
     express: ^4.17.1
     express-promise-router: ^4.1.0
@@ -6821,6 +6822,7 @@ __metadata:
     prom-client: ^15.0.0
     strip-ansi: ^7.1.0
     supertest: ^6.1.3
+    tar: ^6.1.12
     uuid: ^9.0.0
     wait-for-expect: ^3.0.2
     winston: ^3.2.1