diff --git a/plugins/scaffolder-backend-module-gcp-bucket/.eslintrc.js b/plugins/scaffolder-backend-module-gcp-bucket/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp-bucket/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/scaffolder-backend-module-gcp-bucket/CHANGELOG.md b/plugins/scaffolder-backend-module-gcp-bucket/CHANGELOG.md new file mode 100644 index 0000000000..b659b67db4 --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp-bucket/CHANGELOG.md @@ -0,0 +1,3 @@ +# @backstage/plugin-scaffolder-backend-module-gcp-bucket + +## 0.0.1 diff --git a/plugins/scaffolder-backend-module-gcp-bucket/README.md b/plugins/scaffolder-backend-module-gcp-bucket/README.md new file mode 100644 index 0000000000..683dd70c01 --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp-bucket/README.md @@ -0,0 +1,5 @@ +# @backstage/plugin-scaffolder-backend-module-gcp-bucket + +The GCP bucket module for [@backstage/plugin-scaffolder-backend](https://www.npmjs.com/package/@backstage/plugin-scaffolder-backend). + +_This plugin was created through the Backstage CLI_ diff --git a/plugins/scaffolder-backend-module-gcp-bucket/catalog-info.yaml b/plugins/scaffolder-backend-module-gcp-bucket/catalog-info.yaml new file mode 100644 index 0000000000..8fac6a1a2b --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp-bucket/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-scaffolder-backend-module-gcp-bucket + title: '@backstage/plugin-scaffolder-backend-module-gcp-bucket' + description: The GCP bucket module for @backstage/plugin-scaffolder-backend +spec: + lifecycle: experimental + type: backstage-backend-plugin-module + owner: maintainers diff --git a/plugins/scaffolder-backend-module-gcp-bucket/knip-report.md b/plugins/scaffolder-backend-module-gcp-bucket/knip-report.md new file mode 100644 index 0000000000..2661c35327 --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp-bucket/knip-report.md @@ -0,0 +1,2 @@ +# Knip report + diff --git a/plugins/scaffolder-backend-module-gcp-bucket/package.json b/plugins/scaffolder-backend-module-gcp-bucket/package.json new file mode 100644 index 0000000000..426d43df16 --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp-bucket/package.json @@ -0,0 +1,55 @@ +{ + "name": "@backstage/plugin-scaffolder-backend-module-gcp-bucket", + "version": "0.0.1", + "description": "The GCP Bucket module for @backstage/plugin-scaffolder-backend", + "backstage": { + "role": "backend-plugin-module" + }, + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/scaffolder-backend-module-gcp-bucket" + }, + "license": "Apache-2.0", + "exports": { + ".": "./src/index.ts", + "./package.json": "./package.json" + }, + "main": "src/index.ts", + "types": "src/index.ts", + "typesVersions": { + "*": { + "package.json": [ + "package.json" + ] + } + }, + "files": [ + "dist" + ], + "scripts": { + "build": "backstage-cli package build", + "clean": "backstage-cli package clean", + "lint": "backstage-cli package lint", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "start": "backstage-cli package start", + "test": "backstage-cli package test" + }, + "dependencies": { + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^", + "@backstage/integration": "workspace:^", + "@backstage/plugin-scaffolder-node": "workspace:^", + "@google-cloud/storage": "^7.11.2", + "raw-body": "^2.4.1" + }, + "devDependencies": { + "@backstage/cli": "workspace:^", + "@backstage/plugin-scaffolder-node-test-utils": "workspace:^" + } +} diff --git a/plugins/scaffolder-backend-module-gcp-bucket/src/index.ts b/plugins/scaffolder-backend-module-gcp-bucket/src/index.ts new file mode 100644 index 0000000000..05498e3edd --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp-bucket/src/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A module for the scaffolder backend that lets you interact with azure + * + * @packageDocumentation + */ + +export { gcpBucketModule as default } from './module'; diff --git a/plugins/scaffolder-backend-module-gcp-bucket/src/module.ts b/plugins/scaffolder-backend-module-gcp-bucket/src/module.ts new file mode 100644 index 0000000000..3d48b7f708 --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp-bucket/src/module.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { scaffolderWorkspaceProviderExtensionPoint } from '@backstage/plugin-scaffolder-node/alpha'; +import { GcpBucketWorkspaceProvider } from './providers/GcpBucketWorkspaceProvider'; + +/** + * @public + * The Azure Module for the Scaffolder Backend + */ +export const gcpBucketModule = createBackendModule({ + moduleId: 'gcp-bucket', + pluginId: 'scaffolder', + register({ registerInit }) { + registerInit({ + deps: { + scaffolderWorkspaceProviders: scaffolderWorkspaceProviderExtensionPoint, + config: coreServices.rootConfig, + logger: coreServices.logger, + }, + async init({ config, logger, scaffolderWorkspaceProviders }) { + scaffolderWorkspaceProviders.addProviders({ + gcpBucket: GcpBucketWorkspaceProvider.create(logger, config), + }); + }, + }); + }, +}); diff --git a/plugins/scaffolder-backend-module-gcp-bucket/src/providers/GcpBucketWorkspaceProvider.ts b/plugins/scaffolder-backend-module-gcp-bucket/src/providers/GcpBucketWorkspaceProvider.ts new file mode 100644 index 0000000000..8c132e0374 --- /dev/null +++ b/plugins/scaffolder-backend-module-gcp-bucket/src/providers/GcpBucketWorkspaceProvider.ts @@ -0,0 +1,93 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha'; + +import getRawBody from 'raw-body'; +import { Storage } from '@google-cloud/storage'; +import { LoggerService } from '@backstage/backend-plugin-api'; + +import { + serializeWorkspace, + restoreWorkspace, +} from '@backstage/plugin-scaffolder-node'; + +export class GcpBucketWorkspaceProvider implements WorkspaceProvider { + static create(logger: LoggerService, config?: Config) { + return new GcpBucketWorkspaceProvider(new Storage(), logger, config); + } + + private constructor( + private readonly storage: Storage, + private readonly logger: LoggerService, + private readonly config?: Config, + ) {} + + public async cleanWorkspace(options: { taskId: string }): Promise { + await this.storage + .bucket(this.getGcpBucketName()) + .file(options.taskId) + .delete(); + } + + public async serializeWorkspace(options: { + path: string; + taskId: string; + }): Promise { + const fileCloud = this.storage + .bucket(this.getGcpBucketName()) + .file(options.taskId); + const workspace = await serializeWorkspace(options.path); + fileCloud.save( + workspace, + { + contentType: 'application/x-tar', + }, + err => { + if (err) { + this.logger.error( + `An error occurred during uploading the workspace of task ${ + options.taskId + } into GCP bucket ${this.getGcpBucketName()}`, + ); + } + }, + ); + } + + public async rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise { + const bucket = this.storage.bucket(this.getGcpBucketName()); + const file = bucket.file(options.taskId); + const workspace = getRawBody(file.createReadStream()); + await restoreWorkspace(options.targetPath, await workspace); + } + + private getGcpBucketName(): string { + const bucketName = this.config?.getOptionalString( + 'scaffolder.EXPERIMENTAL_gcpBucketName', + ); + if (!bucketName) { + throw new Error( + `You've missed to configure scaffolder.EXPERIMENTAL_gcpBucketName in app-config.yaml file`, + ); + } + return bucketName; + } +} diff --git a/plugins/scaffolder-backend/config.d.ts b/plugins/scaffolder-backend/config.d.ts index 754eedb55b..2e36922355 100644 --- a/plugins/scaffolder-backend/config.d.ts +++ b/plugins/scaffolder-backend/config.d.ts @@ -52,6 +52,15 @@ export interface Config { */ EXPERIMENTAL_workspaceSerialization?: boolean; + /** + * Sets the provider for workspace serialization. + * + * By default, it is your database. + */ + EXPERIMENTAL_workspaceSerializationProvider?: + | 'database' + | 'googleCloudBucket'; + /** * Every task which is in progress state and having a last heartbeat longer than a specified timeout is going to * be attempted to recover. diff --git a/plugins/scaffolder-backend/src/ScaffolderPlugin.ts b/plugins/scaffolder-backend/src/ScaffolderPlugin.ts index c83c6d437d..50e0dff9c0 100644 --- a/plugins/scaffolder-backend/src/ScaffolderPlugin.ts +++ b/plugins/scaffolder-backend/src/ScaffolderPlugin.ts @@ -33,6 +33,8 @@ import { scaffolderAutocompleteExtensionPoint, scaffolderTaskBrokerExtensionPoint, scaffolderTemplatingExtensionPoint, + scaffolderWorkspaceProviderExtensionPoint, + WorkspaceProvider, } from '@backstage/plugin-scaffolder-node/alpha'; import { createCatalogRegisterAction, @@ -84,10 +86,17 @@ export const scaffolderPlugin = createBackendPlugin({ }, }); +<<<<<<< HEAD const autocompleteHandlers: Record = {}; env.registerExtensionPoint(scaffolderAutocompleteExtensionPoint, { addAutocompleteProvider(provider) { autocompleteHandlers[provider.id] = provider.handler; +======= + const additionalWorkspaceProviders: Record = {}; + env.registerExtensionPoint(scaffolderWorkspaceProviderExtensionPoint, { + addProviders(provider) { + Object.assign(additionalWorkspaceProviders, provider); +>>>>>>> 801e5ff2007a (Storing the serialized workspace into google cloud bucket) }, }); @@ -172,6 +181,7 @@ export const scaffolderPlugin = createBackendPlugin({ discovery, permissions, autocompleteHandlers, + additionalWorkspaceProviders, }); httpRouter.use(router); }, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 9c95c11017..02b657032a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -40,7 +40,10 @@ import { DateTime, Duration } from 'luxon'; import { TaskRecovery, TaskSpec } from '@backstage/plugin-scaffolder-common'; import { trimEventsTillLastRecovery } from './taskRecoveryHelper'; import { intervalFromNowTill } from './dbUtil'; -import { restoreWorkspace, serializeWorkspace } from './serializer'; +import { + restoreWorkspace, + serializeWorkspace, +} from '@backstage/plugin-scaffolder-node'; const migrationsDir = resolvePackagePath( '@backstage/plugin-scaffolder-backend', diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseWorkspaceProvider.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseWorkspaceProvider.ts new file mode 100644 index 0000000000..69f90b3d7e --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseWorkspaceProvider.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskStore } from './types'; + +import { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha'; + +export class DatabaseWorkspaceProvider implements WorkspaceProvider { + static create(storage: TaskStore) { + return new DatabaseWorkspaceProvider(storage); + } + + private constructor(private readonly storage: TaskStore) {} + + public async serializeWorkspace(options: { + path: string; + taskId: string; + }): Promise { + this.storage.serializeWorkspace?.(options); + } + + public async rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise { + return this.storage.rehydrateWorkspace?.(options); + } + + public async cleanWorkspace(options: { taskId: string }): Promise { + return this.storage.cleanWorkspace?.(options); + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 4177780e06..f4946e6652 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -35,6 +35,7 @@ import { BackstageCredentials, } from '@backstage/backend-plugin-api'; import { DefaultWorkspaceService, WorkspaceService } from './WorkspaceService'; +import { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha'; type TaskState = { checkpoints: { @@ -66,13 +67,21 @@ export class TaskManager implements TaskContext { logger: Logger, auth?: AuthService, config?: Config, + additionalWorkspaceProviders?: Record, ) { + const workspaceService = DefaultWorkspaceService.create( + task, + storage, + additionalWorkspaceProviders, + config, + ); + const agent = new TaskManager( task, storage, abortSignal, logger, - DefaultWorkspaceService.create(task, storage, config), + workspaceService, auth, ); agent.startTimeout(); @@ -262,6 +271,10 @@ export class StorageTaskBroker implements TaskBroker { private readonly logger: Logger, private readonly config?: Config, private readonly auth?: AuthService, + private readonly additionalWorkspaceProviders?: Record< + string, + WorkspaceProvider + >, ) {} async list(options?: { @@ -350,6 +363,7 @@ export class StorageTaskBroker implements TaskBroker { this.logger, this.auth, this.config, + this.additionalWorkspaceProviders, ); } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/WorkspaceService.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/WorkspaceService.ts index 15de15859d..69deb1c333 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/WorkspaceService.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/WorkspaceService.ts @@ -1,5 +1,5 @@ /* - * Copyright 2021 The Backstage Authors + * Copyright 2024 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,11 @@ * limitations under the License. */ -import { TaskStore } from './types'; - -import { CurrentClaimedTask } from './StorageTaskBroker'; import { Config } from '@backstage/config'; - -type WorkspaceSerializationProvider = 'database' | 'googleCloudBucket'; +import { CurrentClaimedTask } from './StorageTaskBroker'; +import { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha'; +import { DatabaseWorkspaceProvider } from './DatabaseWorkspaceProvider'; +import { TaskStore } from './types'; export interface WorkspaceService { serializeWorkspace(options: { path: string }): Promise; @@ -33,33 +32,40 @@ export interface WorkspaceService { } export class DefaultWorkspaceService implements WorkspaceService { - static create(task: CurrentClaimedTask, storage: TaskStore, config?: Config) { - return new DefaultWorkspaceService(task, storage, config); + static create( + task: CurrentClaimedTask, + storage: TaskStore, + additionalWorkspaceProviders?: Record, + config?: Config, + ) { + const workspaceProviderName = + config?.getOptionalString( + 'scaffolder.EXPERIMENTAL_workspaceSerializationProvider', + ) ?? 'database'; + const workspaceProvider = + additionalWorkspaceProviders?.[workspaceProviderName] ?? + DatabaseWorkspaceProvider.create(storage); + return new DefaultWorkspaceService(task, workspaceProvider); } private constructor( private readonly task: CurrentClaimedTask, - private readonly storage: TaskStore, + private readonly workspaceProvider: WorkspaceProvider, private readonly config?: Config, ) {} public async serializeWorkspace(options: { path: string }): Promise { if (this.isWorkspaceSerializationEnabled()) { - const provider = this.getWorkspaceSerializationProvider(); - switch (provider) { - case 'database': - this.storage.serializeWorkspace?.({ - path: options.path, - taskId: this.task.taskId, - }); - return; - case 'googleCloudBucket': - return; - default: - throw new Error( - `Workspace serialization provider ${provider} is not supported`, - ); - } + await this.workspaceProvider.serializeWorkspace({ + path: options.path, + taskId: this.task.taskId, + }); + } + } + + public async cleanWorkspace(): Promise { + if (this.isWorkspaceSerializationEnabled()) { + await this.workspaceProvider.cleanWorkspace({ taskId: this.task.taskId }); } } @@ -68,12 +74,10 @@ export class DefaultWorkspaceService implements WorkspaceService { targetPath: string; }): Promise { if (this.isWorkspaceSerializationEnabled()) { - this.storage.rehydrateWorkspace?.(options); + await this.workspaceProvider.rehydrateWorkspace(options); } } - public async cleanWorkspace(): Promise {} - private isWorkspaceSerializationEnabled(): boolean { return ( this.config?.getOptionalBoolean( @@ -81,10 +85,4 @@ export class DefaultWorkspaceService implements WorkspaceService { ) ?? false ); } - - private getWorkspaceSerializationProvider(): WorkspaceSerializationProvider { - return (this.config?.getOptionalString( - 'scaffolder.EXPERIMENTAL_workspaceSerializationProvider', - ) ?? 'database') as WorkspaceSerializationProvider; - } } diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 3d5e49b400..769a042a78 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -94,7 +94,10 @@ import { } from '@backstage/plugin-auth-node'; import { InternalTaskSecrets } from '../scaffolder/tasks/types'; import { checkPermission } from '../util/checkPermissions'; -import { AutocompleteHandler } from '@backstage/plugin-scaffolder-node/alpha'; +import { + AutocompleteHandler, + WorkspaceProvider, +} from '@backstage/plugin-scaffolder-node/alpha'; /** * @@ -159,6 +162,7 @@ export interface RouterOptions { taskBroker?: TaskBroker; additionalTemplateFilters?: Record; additionalTemplateGlobals?: Record; + additionalWorkspaceProviders?: Record; permissions?: PermissionsService; permissionRules?: Array< TemplatePermissionRuleInput | ActionPermissionRuleInput @@ -272,6 +276,7 @@ export async function createRouter( scheduler, additionalTemplateFilters, additionalTemplateGlobals, + additionalWorkspaceProviders, permissions, permissionRules, discovery = HostDiscovery.fromConfig(config), @@ -297,7 +302,13 @@ export async function createRouter( let taskBroker: TaskBroker; if (!options.taskBroker) { const databaseTaskStore = await DatabaseTaskStore.create({ database }); - taskBroker = new StorageTaskBroker(databaseTaskStore, logger, config, auth); + taskBroker = new StorageTaskBroker( + databaseTaskStore, + logger, + config, + auth, + additionalWorkspaceProviders, + ); if (scheduler && databaseTaskStore.listStaleTasks) { await scheduler.scheduleTask({ diff --git a/plugins/scaffolder-node/api-report-alpha.md b/plugins/scaffolder-node/api-report-alpha.md index aae32d7203..9601d73215 100644 --- a/plugins/scaffolder-node/api-report-alpha.md +++ b/plugins/scaffolder-node/api-report-alpha.md @@ -68,5 +68,33 @@ export interface ScaffolderTemplatingExtensionPoint { // @alpha export const scaffolderTemplatingExtensionPoint: ExtensionPoint; +// @alpha +export interface ScaffolderWorkspaceProviderExtensionPoint { + // (undocumented) + addProviders(providers: Record): void; +} + +// @alpha +export const scaffolderWorkspaceProviderExtensionPoint: ExtensionPoint; + +// @alpha +export interface WorkspaceProvider { + // (undocumented) + cleanWorkspace(options: { taskId: string }): Promise; + // (undocumented) + rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise; + // (undocumented) + serializeWorkspace({ + path, + taskId, + }: { + path: string; + taskId: string; + }): Promise; +} + // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/scaffolder-node/api-report.md b/plugins/scaffolder-node/api-report.md index 66e8ccd678..9e9a31a991 100644 --- a/plugins/scaffolder-node/api-report.md +++ b/plugins/scaffolder-node/api-report.md @@ -256,6 +256,9 @@ export const parseRepoUrl: ( project?: string | undefined; }; +// @public +export const restoreWorkspace: (path: string, buffer?: Buffer) => Promise; + // @public (undocumented) export interface SerializedFile { // (undocumented) @@ -298,6 +301,9 @@ export type SerializedTaskEvent = { createdAt: string; }; +// @public +export const serializeWorkspace: (path: string) => Promise; + // @public export interface TaskBroker { // (undocumented) diff --git a/plugins/scaffolder-node/package.json b/plugins/scaffolder-node/package.json index ed3c016257..29dc022be1 100644 --- a/plugins/scaffolder-node/package.json +++ b/plugins/scaffolder-node/package.json @@ -61,11 +61,13 @@ "@backstage/integration": "workspace:^", "@backstage/plugin-scaffolder-common": "workspace:^", "@backstage/types": "workspace:^", + "concat-stream": "^2.0.0", "fs-extra": "^11.2.0", "globby": "^11.0.0", "isomorphic-git": "^1.23.0", "jsonschema": "^1.2.6", "p-limit": "^3.1.0", + "tar": "^6.1.12", "winston": "^3.2.1", "zod": "^3.22.4", "zod-to-json-schema": "^3.20.4" diff --git a/plugins/scaffolder-node/src/alpha.ts b/plugins/scaffolder-node/src/alpha.ts index ef4d5ab289..1bb013832c 100644 --- a/plugins/scaffolder-node/src/alpha.ts +++ b/plugins/scaffolder-node/src/alpha.ts @@ -67,6 +67,7 @@ export const scaffolderTaskBrokerExtensionPoint = */ export interface ScaffolderTemplatingExtensionPoint { addTemplateFilters(filters: Record): void; + addTemplateGlobals(filters: Record): void; } @@ -109,7 +110,7 @@ export interface ScaffolderAutocompleteExtensionPoint { } /** - * Extension point for adding template filters and globals. + * Extension point for adding autocomplete handlers. * * @alpha */ @@ -117,3 +118,44 @@ export const scaffolderAutocompleteExtensionPoint = createExtensionPoint({ id: 'scaffolder.autocomplete', }); + +/** + * This provider has to be implemented to make it possible to serialize/deserialize scaffolder workspace. + * + * @alpha + */ +export interface WorkspaceProvider { + serializeWorkspace({ + path, + taskId, + }: { + path: string; + taskId: string; + }): Promise; + + cleanWorkspace(options: { taskId: string }): Promise; + + rehydrateWorkspace(options: { + taskId: string; + targetPath: string; + }): Promise; +} + +/** + * Extension point for adding workspace providers. + * + * @alpha + */ +export interface ScaffolderWorkspaceProviderExtensionPoint { + addProviders(providers: Record): void; +} + +/** + * Extension point for adding workspace providers. + * + * @alpha + */ +export const scaffolderWorkspaceProviderExtensionPoint = + createExtensionPoint({ + id: 'scaffolder.workspace.provider', + }); diff --git a/plugins/scaffolder-node/src/tasks/index.ts b/plugins/scaffolder-node/src/tasks/index.ts index 930de95237..b1754a1f78 100644 --- a/plugins/scaffolder-node/src/tasks/index.ts +++ b/plugins/scaffolder-node/src/tasks/index.ts @@ -26,3 +26,5 @@ export type { TaskEventType, TaskStatus, } from './types'; + +export * from './serializer'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts b/plugins/scaffolder-node/src/tasks/serializer.test.ts similarity index 100% rename from plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts rename to plugins/scaffolder-node/src/tasks/serializer.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts b/plugins/scaffolder-node/src/tasks/serializer.ts similarity index 84% rename from plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts rename to plugins/scaffolder-node/src/tasks/serializer.ts index 69c48fb769..f9a55a68f4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts +++ b/plugins/scaffolder-node/src/tasks/serializer.ts @@ -20,13 +20,22 @@ import { promisify } from 'util'; import { pipeline as pipelineCb, Readable } from 'stream'; const pipeline = promisify(pipelineCb); - +/** + * Serializes provided path into tar archive + * + * @public + */ export const serializeWorkspace = async (path: string): Promise => { - return await new Promise(async resolve => { + return new Promise(async resolve => { await pipeline(tar.create({ cwd: path }, ['']), concatStream(resolve)); }); }; +/** + * Rehydrates the provided buffer of tar archive into the provide destination path + * + * @public + */ export const restoreWorkspace = async (path: string, buffer?: Buffer) => { if (buffer) { await pipeline( diff --git a/yarn.lock b/yarn.lock index e3cec34102..8a1a84b970 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6779,6 +6779,22 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-scaffolder-backend-module-gcp-bucket@workspace:plugins/scaffolder-backend-module-gcp-bucket": + version: 0.0.0-use.local + resolution: "@backstage/plugin-scaffolder-backend-module-gcp-bucket@workspace:plugins/scaffolder-backend-module-gcp-bucket" + dependencies: + "@backstage/backend-plugin-api": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/config": "workspace:^" + "@backstage/errors": "workspace:^" + "@backstage/integration": "workspace:^" + "@backstage/plugin-scaffolder-node": "workspace:^" + "@backstage/plugin-scaffolder-node-test-utils": "workspace:^" + "@google-cloud/storage": ^7.11.2 + raw-body: ^2.4.1 + languageName: unknown + linkType: soft + "@backstage/plugin-scaffolder-backend-module-gerrit@workspace:^, @backstage/plugin-scaffolder-backend-module-gerrit@workspace:plugins/scaffolder-backend-module-gerrit": version: 0.0.0-use.local resolution: "@backstage/plugin-scaffolder-backend-module-gerrit@workspace:plugins/scaffolder-backend-module-gerrit" @@ -7042,11 +7058,13 @@ __metadata: "@backstage/integration": "workspace:^" "@backstage/plugin-scaffolder-common": "workspace:^" "@backstage/types": "workspace:^" + concat-stream: ^2.0.0 fs-extra: ^11.2.0 globby: ^11.0.0 isomorphic-git: ^1.23.0 jsonschema: ^1.2.6 p-limit: ^3.1.0 + tar: ^6.1.12 winston: ^3.2.1 zod: ^3.22.4 zod-to-json-schema: ^3.20.4 @@ -9354,9 +9372,9 @@ __metadata: languageName: node linkType: hard -"@google-cloud/storage@npm:^7.0.0": - version: 7.10.1 - resolution: "@google-cloud/storage@npm:7.10.1" +"@google-cloud/storage@npm:^7.0.0, @google-cloud/storage@npm:^7.11.2": + version: 7.11.2 + resolution: "@google-cloud/storage@npm:7.11.2" dependencies: "@google-cloud/paginator": ^5.0.0 "@google-cloud/projectify": ^4.0.0 @@ -9364,16 +9382,16 @@ __metadata: abort-controller: ^3.0.0 async-retry: ^1.3.3 duplexify: ^4.1.3 - ent: ^2.2.0 fast-xml-parser: ^4.3.0 gaxios: ^6.0.2 google-auth-library: ^9.6.3 + html-entities: ^2.5.2 mime: ^3.0.0 p-limit: ^3.0.1 retry-request: ^7.0.0 teeny-request: ^9.0.0 uuid: ^8.0.0 - checksum: 0605f8abf67cde98bc48b05c749b569416af821fcc3eb8ed3fac6dc5ade9c1f2f3210404c85e30525260ea4f34785d7436f7684e6eacd083613237669badff25 + checksum: 763cd560245006e11a060fdbd5edff5d6bfe22b0c0b06dfe88af43b7b14458598c00a71c4e1b079e9ab09096d7d262bdc27fa77d76a8969af0765f23c93f8405 languageName: node linkType: hard @@ -25281,13 +25299,6 @@ __metadata: languageName: node linkType: hard -"ent@npm:^2.2.0": - version: 2.2.0 - resolution: "ent@npm:2.2.0" - checksum: f588b5707d6fef36011ea10d530645912a69530a1eb0831f8708c498ac028363a7009f45cfadd28ceb4dafd9ac17ec15213f88d09ce239cd033cfe1328dd7d7d - languageName: node - linkType: hard - "entities@npm:^2.0.0": version: 2.2.0 resolution: "entities@npm:2.2.0" @@ -28821,10 +28832,10 @@ __metadata: languageName: node linkType: hard -"html-entities@npm:^2.1.0, html-entities@npm:^2.4.0": - version: 2.4.0 - resolution: "html-entities@npm:2.4.0" - checksum: 25bea32642ce9ebd0eedc4d24381883ecb0335ccb8ac26379a0958b9b16652fdbaa725d70207ce54a51db24103436a698a8e454397d3ba8ad81460224751f1dc +"html-entities@npm:^2.1.0, html-entities@npm:^2.4.0, html-entities@npm:^2.5.2": + version: 2.5.2 + resolution: "html-entities@npm:2.5.2" + checksum: b23f4a07d33d49ade1994069af4e13d31650e3fb62621e92ae10ecdf01d1a98065c78fd20fdc92b4c7881612210b37c275f2c9fba9777650ab0d6f2ceb3b99b6 languageName: node linkType: hard