diff --git a/.changeset/tough-lies-mate.md b/.changeset/tough-lies-mate.md new file mode 100644 index 0000000000..99edcf3616 --- /dev/null +++ b/.changeset/tough-lies-mate.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-scaffolder-backend-module-gcp': minor +'@backstage/plugin-scaffolder-backend': minor +'@backstage/plugin-scaffolder-node': patch +--- + +Serialization of the scaffolder workspace into GCP bucket diff --git a/plugins/scaffolder-backend-module-gcp/.eslintrc.js b/plugins/scaffolder-backend-module-gcp/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/scaffolder-backend-module-gcp/CHANGELOG.md b/plugins/scaffolder-backend-module-gcp/CHANGELOG.md new file mode 100644 index 0000000000..f633c5b8cd --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp/CHANGELOG.md @@ -0,0 +1,3 @@ +# @backstage/plugin-scaffolder-backend-module-gcp + +## 0.0.1 diff --git a/plugins/scaffolder-backend-module-gcp/README.md b/plugins/scaffolder-backend-module-gcp/README.md new file mode 100644 index 0000000000..a9ce86cc6b --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp/README.md @@ -0,0 +1,5 @@ +# @backstage/plugin-scaffolder-backend-module-gcp + +The GCP bucket module for [@backstage/plugin-scaffolder-backend](https://www.npmjs.com/package/@backstage/plugin-scaffolder-backend). + +_This plugin was created through the Backstage CLI_ diff --git a/plugins/scaffolder-backend-module-gcp/api-report.md b/plugins/scaffolder-backend-module-gcp/api-report.md new file mode 100644 index 0000000000..d75a6ce155 --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp/api-report.md @@ -0,0 +1,11 @@ +## API Report File for "@backstage/plugin-scaffolder-backend-module-gcp" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BackendFeatureCompat } from '@backstage/backend-plugin-api'; + +// @public +const gcpBucketModule: BackendFeatureCompat; +export default gcpBucketModule; +``` diff --git a/plugins/scaffolder-backend-module-gcp/catalog-info.yaml b/plugins/scaffolder-backend-module-gcp/catalog-info.yaml new file mode 100644 index 0000000000..175f9860d1 --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-scaffolder-backend-module-gcp + title: '@backstage/plugin-scaffolder-backend-module-gcp' + description: The GCP module for @backstage/plugin-scaffolder-backend +spec: + lifecycle: experimental + type: backstage-backend-plugin-module + owner: maintainers diff --git a/plugins/scaffolder-backend-module-gcp/config.d.ts b/plugins/scaffolder-backend-module-gcp/config.d.ts new file mode 100644 index 0000000000..acd3eebdd9 --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp/config.d.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + /** Configuration options for the scaffolder plugin */ + scaffolder?: { + /** + * Sets GCP bucket name to store serialized workspace for scaffolder tasks. + */ + EXPERIMENTAL_workspaceSerializationGcpBucketName?: string; + }; +} diff --git a/plugins/scaffolder-backend-module-gcp/knip-report.md b/plugins/scaffolder-backend-module-gcp/knip-report.md new file mode 100644 index 0000000000..2661c35327 --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp/knip-report.md @@ -0,0 +1,2 @@ +# Knip report + diff --git a/plugins/scaffolder-backend-module-gcp/package.json b/plugins/scaffolder-backend-module-gcp/package.json new file mode 100644 index 0000000000..7fb3e08b34 --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp/package.json @@ -0,0 +1,59 @@ +{ + "name": "@backstage/plugin-scaffolder-backend-module-gcp", + "version": "0.0.1-next.1", + "description": "The GCP Bucket module for @backstage/plugin-scaffolder-backend", + "backstage": { + "role": "backend-plugin-module", + "pluginId": "scaffolder", + "pluginPackage": "@backstage/plugin-scaffolder-backend" + }, + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/scaffolder-backend-module-gcp" + }, + "license": "Apache-2.0", + "exports": { + ".": "./src/index.ts", + "./package.json": "./package.json" + }, + "main": "src/index.ts", + "types": "src/index.ts", + "typesVersions": { + "*": { + "package.json": [ + "package.json" + ] + } + }, + "files": [ + "dist", + "config.d.ts" + ], + "scripts": { + "build": "backstage-cli package build", + "clean": "backstage-cli package clean", + "lint": "backstage-cli package lint", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "start": "backstage-cli package start", + "test": "backstage-cli package test" + }, + "dependencies": { + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^", + "@backstage/integration": "workspace:^", + "@backstage/plugin-scaffolder-node": "workspace:^", + "@google-cloud/storage": "^7.0.0", + "raw-body": "^2.4.1" + }, + "devDependencies": { + "@backstage/cli": "workspace:^", + "@backstage/plugin-scaffolder-node-test-utils": "workspace:^" + }, + "configSchema": "config.d.ts" +} diff --git a/plugins/scaffolder-backend-module-gcp/src/index.ts b/plugins/scaffolder-backend-module-gcp/src/index.ts new file mode 100644 index 0000000000..05498e3edd --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp/src/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A module for the scaffolder backend that lets you interact with azure + * + * @packageDocumentation + */ + +export { gcpBucketModule as default } from './module'; diff --git a/plugins/scaffolder-backend-module-gcp/src/module.ts b/plugins/scaffolder-backend-module-gcp/src/module.ts new file mode 100644 index 0000000000..0f2f37f5af --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp/src/module.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { scaffolderWorkspaceProviderExtensionPoint } from '@backstage/plugin-scaffolder-node/alpha'; +import { GcpBucketWorkspaceProvider } from './providers/GcpBucketWorkspaceProvider'; + +/** + * @public + * The Azure Module for the Scaffolder Backend + */ +export const gcpBucketModule = createBackendModule({ + moduleId: 'gcp', + pluginId: 'scaffolder', + register({ registerInit }) { + registerInit({ + deps: { + scaffolderWorkspaceProviders: scaffolderWorkspaceProviderExtensionPoint, + config: coreServices.rootConfig, + logger: coreServices.logger, + }, + async init({ config, logger, scaffolderWorkspaceProviders }) { + scaffolderWorkspaceProviders.addProviders({ + gcpBucket: GcpBucketWorkspaceProvider.create(logger, config), + }); + }, + }); + }, +}); diff --git a/plugins/scaffolder-backend-module-gcp/src/providers/GcpBucketWorkspaceProvider.ts b/plugins/scaffolder-backend-module-gcp/src/providers/GcpBucketWorkspaceProvider.ts new file mode 100644 index 0000000000..3529073648 --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp/src/providers/GcpBucketWorkspaceProvider.ts @@ -0,0 +1,99 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha'; + +import getRawBody from 'raw-body'; +import { Storage } from '@google-cloud/storage'; +import { LoggerService } from '@backstage/backend-plugin-api'; + +import { + serializeWorkspace, + restoreWorkspace, +} from '@backstage/plugin-scaffolder-node/alpha'; + +export class GcpBucketWorkspaceProvider implements WorkspaceProvider { + static create(logger: LoggerService, config?: Config) { + return new GcpBucketWorkspaceProvider(new Storage(), logger, config); + } + + private constructor( + private readonly storage: Storage, + private readonly logger: LoggerService, + private readonly config?: Config, + ) {} + + public async cleanWorkspace(options: { taskId: string }): Promise { + const file = this.storage + .bucket(this.getGcpBucketName()) + .file(options.taskId); + + const result = await file.exists(); + if (result[0]) { + await file.delete(); + } + } + + public async serializeWorkspace(options: { + path: string; + taskId: string; + }): Promise { + const fileCloud = this.storage + .bucket(this.getGcpBucketName()) + .file(options.taskId); + const { contents: workspace } = await serializeWorkspace(options); + try { + await fileCloud.save(workspace, { + contentType: 'application/x-tar', + }); + } catch (err) { + this.logger.error( + `An error occurred during uploading the workspace of task ${ + options.taskId + } into GCP bucket ${this.getGcpBucketName()}`, + ); + } + this.logger.info( + `Workspace for task ${options.taskId} has been serialized.`, + ); + } + + public async rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise { + const bucket = this.storage.bucket(this.getGcpBucketName()); + const file = bucket.file(options.taskId); + const result = await file.exists(); + if (result[0]) { + const workspace = await getRawBody(file.createReadStream()); + await restoreWorkspace({ path: options.targetPath, buffer: workspace }); + } + } + + private getGcpBucketName(): string { + const bucketName = this.config?.getOptionalString( + 'scaffolder.EXPERIMENTAL_workspaceSerializationGcpBucketName', + ); + if (!bucketName) { + throw new Error( + `You've missed to configure scaffolder.EXPERIMENTAL_workspaceSerializationGcpBucketName in app-config.yaml file`, + ); + } + return bucketName; + } +} diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 4d75ed8421..a3beee2b53 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -64,6 +64,7 @@ import { TemplateFilter as TemplateFilter_2 } from '@backstage/plugin-scaffolder import { TemplateGlobal as TemplateGlobal_2 } from '@backstage/plugin-scaffolder-node'; import { TemplateParametersV1beta3 } from '@backstage/plugin-scaffolder-common'; import { UrlReader } from '@backstage/backend-common'; +import { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha'; import { ZodType } from 'zod'; import { ZodTypeDef } from 'zod'; @@ -477,6 +478,8 @@ export interface RouterOptions { // (undocumented) additionalTemplateGlobals?: Record; // (undocumented) + additionalWorkspaceProviders?: Record; + // (undocumented) auth?: AuthService; // (undocumented) autocompleteHandlers?: Record; @@ -559,6 +562,7 @@ export class TaskManager implements TaskContext_2 { logger: Logger, auth?: AuthService, config?: Config, + additionalWorkspaceProviders?: Record, ): TaskManager; // (undocumented) get createdBy(): string | undefined; diff --git a/plugins/scaffolder-backend/config.d.ts b/plugins/scaffolder-backend/config.d.ts index 754eedb55b..f6b9522575 100644 --- a/plugins/scaffolder-backend/config.d.ts +++ b/plugins/scaffolder-backend/config.d.ts @@ -52,6 +52,13 @@ export interface Config { */ EXPERIMENTAL_workspaceSerialization?: boolean; + /** + * Sets the provider for workspace serialization. + * + * By default, it is your database. + */ + EXPERIMENTAL_workspaceSerializationProvider?: string; + /** * Every task which is in progress state and having a last heartbeat longer than a specified timeout is going to * be attempted to recover. diff --git a/plugins/scaffolder-backend/src/ScaffolderPlugin.ts b/plugins/scaffolder-backend/src/ScaffolderPlugin.ts index c83c6d437d..27b4f6b196 100644 --- a/plugins/scaffolder-backend/src/ScaffolderPlugin.ts +++ b/plugins/scaffolder-backend/src/ScaffolderPlugin.ts @@ -33,6 +33,8 @@ import { scaffolderAutocompleteExtensionPoint, scaffolderTaskBrokerExtensionPoint, scaffolderTemplatingExtensionPoint, + scaffolderWorkspaceProviderExtensionPoint, + WorkspaceProvider, } from '@backstage/plugin-scaffolder-node/alpha'; import { createCatalogRegisterAction, @@ -91,6 +93,13 @@ export const scaffolderPlugin = createBackendPlugin({ }, }); + const additionalWorkspaceProviders: Record = {}; + env.registerExtensionPoint(scaffolderWorkspaceProviderExtensionPoint, { + addProviders(provider) { + Object.assign(additionalWorkspaceProviders, provider); + }, + }); + env.registerInit({ deps: { logger: coreServices.logger, @@ -172,6 +181,7 @@ export const scaffolderPlugin = createBackendPlugin({ discovery, permissions, autocompleteHandlers, + additionalWorkspaceProviders, }); httpRouter.use(router); }, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 9c95c11017..fe62572af9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -40,7 +40,10 @@ import { DateTime, Duration } from 'luxon'; import { TaskRecovery, TaskSpec } from '@backstage/plugin-scaffolder-common'; import { trimEventsTillLastRecovery } from './taskRecoveryHelper'; import { intervalFromNowTill } from './dbUtil'; -import { restoreWorkspace, serializeWorkspace } from './serializer'; +import { + restoreWorkspace, + serializeWorkspace, +} from '@backstage/plugin-scaffolder-node/alpha'; const migrationsDir = resolvePackagePath( '@backstage/plugin-scaffolder-backend', @@ -517,7 +520,10 @@ export class DatabaseTaskStore implements TaskStore { .where({ id: options.taskId }) .select('workspace'); - await restoreWorkspace(options.targetPath, result.workspace); + await restoreWorkspace({ + path: options.targetPath, + buffer: result.workspace, + }); } async cleanWorkspace({ taskId }: { taskId: string }): Promise { @@ -534,7 +540,7 @@ export class DatabaseTaskStore implements TaskStore { await this.db('tasks') .where({ id: options.taskId }) .update({ - workspace: await serializeWorkspace(options.path), + workspace: (await serializeWorkspace(options)).contents, }); } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseWorkspaceProvider.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseWorkspaceProvider.ts new file mode 100644 index 0000000000..69f90b3d7e --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseWorkspaceProvider.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskStore } from './types'; + +import { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha'; + +export class DatabaseWorkspaceProvider implements WorkspaceProvider { + static create(storage: TaskStore) { + return new DatabaseWorkspaceProvider(storage); + } + + private constructor(private readonly storage: TaskStore) {} + + public async serializeWorkspace(options: { + path: string; + taskId: string; + }): Promise { + this.storage.serializeWorkspace?.(options); + } + + public async rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise { + return this.storage.rehydrateWorkspace?.(options); + } + + public async cleanWorkspace(options: { taskId: string }): Promise { + return this.storage.cleanWorkspace?.(options); + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index e9b1d78e36..3b88fd0f10 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -443,7 +443,6 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { throw new Error(`Step ${step.name} has been cancelled.`); } - await task.cleanWorkspace?.(); await stepTrack.markSuccessful(); } catch (err) { await taskTrack.markFailed(step, err); @@ -513,6 +512,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { return { output }; } finally { if (workspacePath) { + await task.cleanWorkspace?.(); await fs.remove(workspacePath); } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 03a2833938..f4946e6652 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -34,6 +34,8 @@ import { AuthService, BackstageCredentials, } from '@backstage/backend-plugin-api'; +import { DefaultWorkspaceService, WorkspaceService } from './WorkspaceService'; +import { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha'; type TaskState = { checkpoints: { @@ -65,14 +67,22 @@ export class TaskManager implements TaskContext { logger: Logger, auth?: AuthService, config?: Config, + additionalWorkspaceProviders?: Record, ) { + const workspaceService = DefaultWorkspaceService.create( + task, + storage, + additionalWorkspaceProviders, + config, + ); + const agent = new TaskManager( task, storage, abortSignal, logger, + workspaceService, auth, - config, ); agent.startTimeout(); return agent; @@ -84,8 +94,8 @@ export class TaskManager implements TaskContext { private readonly storage: TaskStore, private readonly signal: AbortSignal, private readonly logger: Logger, + private readonly workspaceService: WorkspaceService, private readonly auth?: AuthService, - private readonly config?: Config, ) {} get spec() { @@ -112,9 +122,7 @@ export class TaskManager implements TaskContext { taskId: string; targetPath: string; }): Promise { - if (this.isWorkspaceSerializationEnabled()) { - this.storage.rehydrateWorkspace?.(options); - } + await this.workspaceService.rehydrateWorkspace(options); } get done() { @@ -163,18 +171,11 @@ export class TaskManager implements TaskContext { } async serializeWorkspace?(options: { path: string }): Promise { - if (this.isWorkspaceSerializationEnabled()) { - await this.storage.serializeWorkspace?.({ - path: options.path, - taskId: this.task.taskId, - }); - } + await this.workspaceService.serializeWorkspace(options); } async cleanWorkspace?(): Promise { - if (this.isWorkspaceSerializationEnabled()) { - await this.storage.cleanWorkspace?.({ taskId: this.task.taskId }); - } + await this.workspaceService.cleanWorkspace(); } async complete( @@ -211,14 +212,6 @@ export class TaskManager implements TaskContext { }, 1000); } - private isWorkspaceSerializationEnabled(): boolean { - return ( - this.config?.getOptionalBoolean( - 'scaffolder.EXPERIMENTAL_workspaceSerialization', - ) ?? false - ); - } - async getInitiatorCredentials(): Promise { const secrets = this.task.secrets as InternalTaskSecrets; @@ -278,6 +271,10 @@ export class StorageTaskBroker implements TaskBroker { private readonly logger: Logger, private readonly config?: Config, private readonly auth?: AuthService, + private readonly additionalWorkspaceProviders?: Record< + string, + WorkspaceProvider + >, ) {} async list(options?: { @@ -366,6 +363,7 @@ export class StorageTaskBroker implements TaskBroker { this.logger, this.auth, this.config, + this.additionalWorkspaceProviders, ); } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/WorkspaceService.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/WorkspaceService.ts new file mode 100644 index 0000000000..ab424806bf --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/WorkspaceService.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { CurrentClaimedTask } from './StorageTaskBroker'; +import { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha'; +import { DatabaseWorkspaceProvider } from './DatabaseWorkspaceProvider'; +import { TaskStore } from './types'; + +export interface WorkspaceService { + serializeWorkspace(options: { path: string }): Promise; + + cleanWorkspace(): Promise; + + rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise; +} + +export class DefaultWorkspaceService implements WorkspaceService { + static create( + task: CurrentClaimedTask, + storage: TaskStore, + additionalWorkspaceProviders?: Record, + config?: Config, + ) { + const workspaceProviderName = + config?.getOptionalString( + 'scaffolder.EXPERIMENTAL_workspaceSerializationProvider', + ) ?? 'database'; + const workspaceProvider = + additionalWorkspaceProviders?.[workspaceProviderName] ?? + DatabaseWorkspaceProvider.create(storage); + return new DefaultWorkspaceService(task, workspaceProvider, config); + } + + private constructor( + private readonly task: CurrentClaimedTask, + private readonly workspaceProvider: WorkspaceProvider, + private readonly config?: Config, + ) {} + + public async serializeWorkspace(options: { path: string }): Promise { + if (this.isWorkspaceSerializationEnabled()) { + await this.workspaceProvider.serializeWorkspace({ + path: options.path, + taskId: this.task.taskId, + }); + } + } + + public async cleanWorkspace(): Promise { + if (this.isWorkspaceSerializationEnabled()) { + await this.workspaceProvider.cleanWorkspace({ taskId: this.task.taskId }); + } + } + + public async rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise { + if (this.isWorkspaceSerializationEnabled()) { + await this.workspaceProvider.rehydrateWorkspace(options); + } + } + + private isWorkspaceSerializationEnabled(): boolean { + return ( + this.config?.getOptionalBoolean( + 'scaffolder.EXPERIMENTAL_workspaceSerialization', + ) ?? false + ); + } +} diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 3d5e49b400..769a042a78 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -94,7 +94,10 @@ import { } from '@backstage/plugin-auth-node'; import { InternalTaskSecrets } from '../scaffolder/tasks/types'; import { checkPermission } from '../util/checkPermissions'; -import { AutocompleteHandler } from '@backstage/plugin-scaffolder-node/alpha'; +import { + AutocompleteHandler, + WorkspaceProvider, +} from '@backstage/plugin-scaffolder-node/alpha'; /** * @@ -159,6 +162,7 @@ export interface RouterOptions { taskBroker?: TaskBroker; additionalTemplateFilters?: Record; additionalTemplateGlobals?: Record; + additionalWorkspaceProviders?: Record; permissions?: PermissionsService; permissionRules?: Array< TemplatePermissionRuleInput | ActionPermissionRuleInput @@ -272,6 +276,7 @@ export async function createRouter( scheduler, additionalTemplateFilters, additionalTemplateGlobals, + additionalWorkspaceProviders, permissions, permissionRules, discovery = HostDiscovery.fromConfig(config), @@ -297,7 +302,13 @@ export async function createRouter( let taskBroker: TaskBroker; if (!options.taskBroker) { const databaseTaskStore = await DatabaseTaskStore.create({ database }); - taskBroker = new StorageTaskBroker(databaseTaskStore, logger, config, auth); + taskBroker = new StorageTaskBroker( + databaseTaskStore, + logger, + config, + auth, + additionalWorkspaceProviders, + ); if (scheduler && databaseTaskStore.listStaleTasks) { await scheduler.scheduleTask({ diff --git a/plugins/scaffolder-node/api-report-alpha.md b/plugins/scaffolder-node/api-report-alpha.md index aae32d7203..c9b8130f0e 100644 --- a/plugins/scaffolder-node/api-report-alpha.md +++ b/plugins/scaffolder-node/api-report-alpha.md @@ -3,6 +3,8 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +/// + import { ExtensionPoint } from '@backstage/backend-plugin-api'; import { TaskBroker } from '@backstage/plugin-scaffolder-node'; import { TemplateAction } from '@backstage/plugin-scaffolder-node'; @@ -24,6 +26,12 @@ export type AutocompleteHandler = ({ }[]; }>; +// @alpha +export const restoreWorkspace: (opts: { + path: string; + buffer?: Buffer; +}) => Promise; + // @alpha export interface ScaffolderActionsExtensionPoint { // (undocumented) @@ -68,5 +76,38 @@ export interface ScaffolderTemplatingExtensionPoint { // @alpha export const scaffolderTemplatingExtensionPoint: ExtensionPoint; +// @alpha +export interface ScaffolderWorkspaceProviderExtensionPoint { + // (undocumented) + addProviders(providers: Record): void; +} + +// @alpha +export const scaffolderWorkspaceProviderExtensionPoint: ExtensionPoint; + +// @alpha +export const serializeWorkspace: (opts: { path: string }) => Promise<{ + contents: Buffer; +}>; + +// @alpha +export interface WorkspaceProvider { + // (undocumented) + cleanWorkspace(options: { taskId: string }): Promise; + // (undocumented) + rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise; + // (undocumented) + serializeWorkspace({ + path, + taskId, + }: { + path: string; + taskId: string; + }): Promise; +} + // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/scaffolder-node/package.json b/plugins/scaffolder-node/package.json index ed3c016257..29dc022be1 100644 --- a/plugins/scaffolder-node/package.json +++ b/plugins/scaffolder-node/package.json @@ -61,11 +61,13 @@ "@backstage/integration": "workspace:^", "@backstage/plugin-scaffolder-common": "workspace:^", "@backstage/types": "workspace:^", + "concat-stream": "^2.0.0", "fs-extra": "^11.2.0", "globby": "^11.0.0", "isomorphic-git": "^1.23.0", "jsonschema": "^1.2.6", "p-limit": "^3.1.0", + "tar": "^6.1.12", "winston": "^3.2.1", "zod": "^3.22.4", "zod-to-json-schema": "^3.20.4" diff --git a/plugins/scaffolder-node/src/alpha.ts b/plugins/scaffolder-node/src/alpha.ts index ef4d5ab289..6a0e34faf7 100644 --- a/plugins/scaffolder-node/src/alpha.ts +++ b/plugins/scaffolder-node/src/alpha.ts @@ -22,6 +22,8 @@ import { TaskBroker, } from '@backstage/plugin-scaffolder-node'; +export * from './tasks/alpha'; + /** * Extension point for managing scaffolder actions. * @@ -67,6 +69,7 @@ export const scaffolderTaskBrokerExtensionPoint = */ export interface ScaffolderTemplatingExtensionPoint { addTemplateFilters(filters: Record): void; + addTemplateGlobals(filters: Record): void; } @@ -109,7 +112,7 @@ export interface ScaffolderAutocompleteExtensionPoint { } /** - * Extension point for adding template filters and globals. + * Extension point for adding autocomplete handlers. * * @alpha */ @@ -117,3 +120,44 @@ export const scaffolderAutocompleteExtensionPoint = createExtensionPoint({ id: 'scaffolder.autocomplete', }); + +/** + * This provider has to be implemented to make it possible to serialize/deserialize scaffolder workspace. + * + * @alpha + */ +export interface WorkspaceProvider { + serializeWorkspace({ + path, + taskId, + }: { + path: string; + taskId: string; + }): Promise; + + cleanWorkspace(options: { taskId: string }): Promise; + + rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise; +} + +/** + * Extension point for adding workspace providers. + * + * @alpha + */ +export interface ScaffolderWorkspaceProviderExtensionPoint { + addProviders(providers: Record): void; +} + +/** + * Extension point for adding workspace providers. + * + * @alpha + */ +export const scaffolderWorkspaceProviderExtensionPoint = + createExtensionPoint({ + id: 'scaffolder.workspace.provider', + }); diff --git a/plugins/scaffolder-node/src/tasks/alpha.ts b/plugins/scaffolder-node/src/tasks/alpha.ts new file mode 100644 index 0000000000..fb6b09026c --- /dev/null +++ b/plugins/scaffolder-node/src/tasks/alpha.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export * from './serializer'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts b/plugins/scaffolder-node/src/tasks/serializer.test.ts similarity index 96% rename from plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts rename to plugins/scaffolder-node/src/tasks/serializer.test.ts index bec5398e28..a0ddfa624d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts +++ b/plugins/scaffolder-node/src/tasks/serializer.test.ts @@ -72,8 +72,11 @@ describe('serializer', () => { const restoredWorkspaceDir = createMockDirectory(); it('should be able to archive and restore the workspace', async () => { - const workspaceBuffer = await serializeWorkspace(workspaceDir.path); - await restoreWorkspace(restoredWorkspaceDir.path, workspaceBuffer); + const workspaceBuffer = await serializeWorkspace(workspaceDir); + await restoreWorkspace({ + path: restoredWorkspaceDir.path, + buffer: workspaceBuffer.contents, + }); expect( fs.existsSync(`${restoredWorkspaceDir.path}/\$\{ESCAPE_ME\}.txt`), diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts b/plugins/scaffolder-node/src/tasks/serializer.ts similarity index 59% rename from plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts rename to plugins/scaffolder-node/src/tasks/serializer.ts index 69c48fb769..baf1b42157 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts +++ b/plugins/scaffolder-node/src/tasks/serializer.ts @@ -20,14 +20,34 @@ import { promisify } from 'util'; import { pipeline as pipelineCb, Readable } from 'stream'; const pipeline = promisify(pipelineCb); - -export const serializeWorkspace = async (path: string): Promise => { - return await new Promise(async resolve => { - await pipeline(tar.create({ cwd: path }, ['']), concatStream(resolve)); +/** + * Serializes provided path into tar archive + * + * @alpha + */ +export const serializeWorkspace = async (opts: { + path: string; +}): Promise<{ contents: Buffer }> => { + return new Promise<{ contents: Buffer }>(async resolve => { + await pipeline( + tar.create({ cwd: opts.path }, ['']), + concatStream(buffer => { + return resolve({ contents: buffer }); + }), + ); }); }; -export const restoreWorkspace = async (path: string, buffer?: Buffer) => { +/** + * Rehydrates the provided buffer of tar archive into the provide destination path + * + * @alpha + */ +export const restoreWorkspace = async (opts: { + path: string; + buffer?: Buffer; +}): Promise => { + const { buffer, path } = opts; if (buffer) { await pipeline( Readable.from(buffer), diff --git a/yarn.lock b/yarn.lock index e3cec34102..3671d290fc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6779,6 +6779,22 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-scaffolder-backend-module-gcp@workspace:plugins/scaffolder-backend-module-gcp": + version: 0.0.0-use.local + resolution: "@backstage/plugin-scaffolder-backend-module-gcp@workspace:plugins/scaffolder-backend-module-gcp" + dependencies: + "@backstage/backend-plugin-api": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/config": "workspace:^" + "@backstage/errors": "workspace:^" + "@backstage/integration": "workspace:^" + "@backstage/plugin-scaffolder-node": "workspace:^" + "@backstage/plugin-scaffolder-node-test-utils": "workspace:^" + "@google-cloud/storage": ^7.0.0 + raw-body: ^2.4.1 + languageName: unknown + linkType: soft + "@backstage/plugin-scaffolder-backend-module-gerrit@workspace:^, @backstage/plugin-scaffolder-backend-module-gerrit@workspace:plugins/scaffolder-backend-module-gerrit": version: 0.0.0-use.local resolution: "@backstage/plugin-scaffolder-backend-module-gerrit@workspace:plugins/scaffolder-backend-module-gerrit" @@ -7042,11 +7058,13 @@ __metadata: "@backstage/integration": "workspace:^" "@backstage/plugin-scaffolder-common": "workspace:^" "@backstage/types": "workspace:^" + concat-stream: ^2.0.0 fs-extra: ^11.2.0 globby: ^11.0.0 isomorphic-git: ^1.23.0 jsonschema: ^1.2.6 p-limit: ^3.1.0 + tar: ^6.1.12 winston: ^3.2.1 zod: ^3.22.4 zod-to-json-schema: ^3.20.4 @@ -9355,8 +9373,8 @@ __metadata: linkType: hard "@google-cloud/storage@npm:^7.0.0": - version: 7.10.1 - resolution: "@google-cloud/storage@npm:7.10.1" + version: 7.11.2 + resolution: "@google-cloud/storage@npm:7.11.2" dependencies: "@google-cloud/paginator": ^5.0.0 "@google-cloud/projectify": ^4.0.0 @@ -9364,16 +9382,16 @@ __metadata: abort-controller: ^3.0.0 async-retry: ^1.3.3 duplexify: ^4.1.3 - ent: ^2.2.0 fast-xml-parser: ^4.3.0 gaxios: ^6.0.2 google-auth-library: ^9.6.3 + html-entities: ^2.5.2 mime: ^3.0.0 p-limit: ^3.0.1 retry-request: ^7.0.0 teeny-request: ^9.0.0 uuid: ^8.0.0 - checksum: 0605f8abf67cde98bc48b05c749b569416af821fcc3eb8ed3fac6dc5ade9c1f2f3210404c85e30525260ea4f34785d7436f7684e6eacd083613237669badff25 + checksum: 763cd560245006e11a060fdbd5edff5d6bfe22b0c0b06dfe88af43b7b14458598c00a71c4e1b079e9ab09096d7d262bdc27fa77d76a8969af0765f23c93f8405 languageName: node linkType: hard @@ -25281,13 +25299,6 @@ __metadata: languageName: node linkType: hard -"ent@npm:^2.2.0": - version: 2.2.0 - resolution: "ent@npm:2.2.0" - checksum: f588b5707d6fef36011ea10d530645912a69530a1eb0831f8708c498ac028363a7009f45cfadd28ceb4dafd9ac17ec15213f88d09ce239cd033cfe1328dd7d7d - languageName: node - linkType: hard - "entities@npm:^2.0.0": version: 2.2.0 resolution: "entities@npm:2.2.0" @@ -28821,10 +28832,10 @@ __metadata: languageName: node linkType: hard -"html-entities@npm:^2.1.0, html-entities@npm:^2.4.0": - version: 2.4.0 - resolution: "html-entities@npm:2.4.0" - checksum: 25bea32642ce9ebd0eedc4d24381883ecb0335ccb8ac26379a0958b9b16652fdbaa725d70207ce54a51db24103436a698a8e454397d3ba8ad81460224751f1dc +"html-entities@npm:^2.1.0, html-entities@npm:^2.4.0, html-entities@npm:^2.5.2": + version: 2.5.2 + resolution: "html-entities@npm:2.5.2" + checksum: b23f4a07d33d49ade1994069af4e13d31650e3fb62621e92ae10ecdf01d1a98065c78fd20fdc92b4c7881612210b37c275f2c9fba9777650ab0d6f2ceb3b99b6 languageName: node linkType: hard