Storing the serialized workspace into google cloud bucket

Signed-off-by: bnechyporenko <bnechyporenko@bol.com>

plugins/scaffolder-node/api-report-alpha.md

@@ -68,5 +68,33 @@ export interface ScaffolderTemplatingExtensionPoint {
 // @alpha
 export const scaffolderTemplatingExtensionPoint: ExtensionPoint<ScaffolderTemplatingExtensionPoint>;
 
+// @alpha
+export interface ScaffolderWorkspaceProviderExtensionPoint {
+  // (undocumented)
+  addProviders(providers: Record<string, WorkspaceProvider>): void;
+}
+
+// @alpha
+export const scaffolderWorkspaceProviderExtensionPoint: ExtensionPoint<ScaffolderWorkspaceProviderExtensionPoint>;
+
+// @alpha
+export interface WorkspaceProvider {
+  // (undocumented)
+  cleanWorkspace(options: { taskId: string }): Promise<void>;
+  // (undocumented)
+  rehydrateWorkspace(options: {
+    taskId: string;
+    targetPath: string;
+  }): Promise<void>;
+  // (undocumented)
+  serializeWorkspace({
+    path,
+    taskId,
+  }: {
+    path: string;
+    taskId: string;
+  }): Promise<void>;
+}
+
 // (No @packageDocumentation comment for this package)
 ```

plugins/scaffolder-node/api-report.md

@@ -256,6 +256,9 @@ export const parseRepoUrl: (
   project?: string | undefined;
 };
 
+// @public
+export const restoreWorkspace: (path: string, buffer?: Buffer) => Promise<void>;
+
 // @public (undocumented)
 export interface SerializedFile {
   // (undocumented)
@@ -298,6 +301,9 @@ export type SerializedTaskEvent = {
   createdAt: string;
 };
 
+// @public
+export const serializeWorkspace: (path: string) => Promise<Buffer>;
+
 // @public
 export interface TaskBroker {
   // (undocumented)

plugins/scaffolder-node/package.json

@@ -61,11 +61,13 @@
     "@backstage/integration": "workspace:^",
     "@backstage/plugin-scaffolder-common": "workspace:^",
     "@backstage/types": "workspace:^",
+    "concat-stream": "^2.0.0",
     "fs-extra": "^11.2.0",
     "globby": "^11.0.0",
     "isomorphic-git": "^1.23.0",
     "jsonschema": "^1.2.6",
     "p-limit": "^3.1.0",
+    "tar": "^6.1.12",
     "winston": "^3.2.1",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.20.4"

plugins/scaffolder-node/src/alpha.ts

@@ -67,6 +67,7 @@ export const scaffolderTaskBrokerExtensionPoint =
  */
 export interface ScaffolderTemplatingExtensionPoint {
   addTemplateFilters(filters: Record<string, TemplateFilter>): void;
+
   addTemplateGlobals(filters: Record<string, TemplateGlobal>): void;
 }
 
@@ -109,7 +110,7 @@ export interface ScaffolderAutocompleteExtensionPoint {
 }
 
 /**
- * Extension point for adding template filters and globals.
+ * Extension point for adding autocomplete handlers.
  *
  * @alpha
  */
@@ -117,3 +118,44 @@ export const scaffolderAutocompleteExtensionPoint =
   createExtensionPoint<ScaffolderAutocompleteExtensionPoint>({
     id: 'scaffolder.autocomplete',
   });
+
+/**
+ * This provider has to be implemented to make it possible to serialize/deserialize scaffolder workspace.
+ *
+ * @alpha
+ */
+export interface WorkspaceProvider {
+  serializeWorkspace({
+    path,
+    taskId,
+  }: {
+    path: string;
+    taskId: string;
+  }): Promise<void>;
+
+  cleanWorkspace(options: { taskId: string }): Promise<void>;
+
+  rehydrateWorkspace(options: {
+    taskId: string;
+    targetPath: string;
+  }): Promise<void>;
+}
+
+/**
+ * Extension point for adding workspace providers.
+ *
+ * @alpha
+ */
+export interface ScaffolderWorkspaceProviderExtensionPoint {
+  addProviders(providers: Record<string, WorkspaceProvider>): void;
+}
+
+/**
+ * Extension point for adding workspace providers.
+ *
+ * @alpha
+ */
+export const scaffolderWorkspaceProviderExtensionPoint =
+  createExtensionPoint<ScaffolderWorkspaceProviderExtensionPoint>({
+    id: 'scaffolder.workspace.provider',
+  });

plugins/scaffolder-node/src/tasks/index.ts

@@ -26,3 +26,5 @@ export type {
   TaskEventType,
   TaskStatus,
 } from './types';
+
+export * from './serializer';

plugins/scaffolder-node/src/tasks/serializer.test.ts

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { serializeWorkspace, restoreWorkspace } from './serializer';
+import { createMockDirectory } from '@backstage/backend-test-utils';
+import fs from 'fs-extra';
+
+describe('serializer', () => {
+  const workspaceDir = createMockDirectory({
+    content: {
+      'app-config.yaml': `
+            app:
+              title: Example App
+              sessionKey:
+                $file: secrets/session-key.txt
+              escaped: \$\${Escaped}
+          `,
+      'app-config2.yaml': `
+            app:
+              title: Example App 2
+              sessionKey:
+                $file: secrets/session-key.txt
+              escaped: \$\${Escaped}
+          `,
+      'app-config.development.yaml': `
+            app:
+              sessionKey: development-key
+            backend:
+              $include: ./included.yaml
+            other:
+              $include: secrets/included.yaml
+          `,
+      'secrets/session-key.txt': 'abc123',
+      'secrets/included.yaml': `
+            secret:
+              $file: session-key.txt
+          `,
+      'included.yaml': `
+            foo:
+              bar: token \${MY_SECRET}
+          `,
+      'app-config.substitute.yaml': `
+            app:
+              someConfig:
+                $include: \${SUBSTITUTE_ME}.yaml
+              noSubstitute:
+                $file: \$\${ESCAPE_ME}.txt
+          `,
+      'substituted.yaml': `
+            secret:
+              $file: secrets/\${SUBSTITUTE_ME}.txt
+          `,
+      'secrets/substituted.txt': '123abc',
+      '${ESCAPE_ME}.txt': 'notSubstituted',
+      'empty.yaml': '# just a comment',
+    },
+  });
+
+  const restoredWorkspaceDir = createMockDirectory();
+
+  it('should be able to archive and restore the workspace', async () => {
+    const workspaceBuffer = await serializeWorkspace(workspaceDir.path);
+    await restoreWorkspace(restoredWorkspaceDir.path, workspaceBuffer);
+
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/\$\{ESCAPE_ME\}.txt`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/app-config.development.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/app-config.substitute.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/app-config.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/app-config2.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/empty.yaml`),
+    ).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/included.yaml`),
+    ).toBeTruthy();
+    expect(fs.existsSync(`${restoredWorkspaceDir.path}/secrets`)).toBeTruthy();
+    expect(
+      fs.existsSync(`${restoredWorkspaceDir.path}/substituted.yaml`),
+    ).toBeTruthy();
+
+    expect(
+      fs.readFileSync(`${restoredWorkspaceDir.path}/substituted.yaml`, 'utf8'),
+    ).toEqual(`
+            secret:
+              $file: secrets/\${SUBSTITUTE_ME}.txt
+          `);
+  });
+});

plugins/scaffolder-node/src/tasks/serializer.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import tar from 'tar';
+import concatStream from 'concat-stream';
+import { promisify } from 'util';
+import { pipeline as pipelineCb, Readable } from 'stream';
+
+const pipeline = promisify(pipelineCb);
+/**
+ * Serializes provided path into tar archive
+ *
+ * @public
+ */
+export const serializeWorkspace = async (path: string): Promise<Buffer> => {
+  return new Promise<Buffer>(async resolve => {
+    await pipeline(tar.create({ cwd: path }, ['']), concatStream(resolve));
+  });
+};
+
+/**
+ * Rehydrates the provided buffer of tar archive into the provide destination path
+ *
+ * @public
+ */
+export const restoreWorkspace = async (path: string, buffer?: Buffer) => {
+  if (buffer) {
+    await pipeline(
+      Readable.from(buffer),
+      tar.extract({
+        C: path,
+      }),
+    );
+  }
+};