paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 41 Packages Projects Releases Wiki Activity

Merge pull request #25142 from acierto/serder-bucket

Browse Source 
Serialization of the scaffolder workspace into GCP bucket
This commit is contained in:
Ben Lambert
2024-06-25 15:00:10 +02:00
committed by GitHub
parent 97e247a62d b61503b2ba
commit 24f91f6d77
28 changed files with 646 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/tough-lies-mate.md
+7
View File
@@ -0,0 +1,7 @@
---
'@backstage/plugin-scaffolder-backend-module-gcp': minor
'@backstage/plugin-scaffolder-backend': minor
'@backstage/plugin-scaffolder-node': patch
---
Serialization of the scaffolder workspace into GCP bucket
plugins/scaffolder-backend-module-gcp/.eslintrc.js
+1
View File
@@ -0,0 +1 @@
module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);
plugins/scaffolder-backend-module-gcp/CHANGELOG.md
+3
View File
@@ -0,0 +1,3 @@
# @backstage/plugin-scaffolder-backend-module-gcp
## 0.0.1
plugins/scaffolder-backend-module-gcp/README.md
+5
View File
@@ -0,0 +1,5 @@
# @backstage/plugin-scaffolder-backend-module-gcp
The GCP bucket module for [@backstage/plugin-scaffolder-backend](https://www.npmjs.com/package/@backstage/plugin-scaffolder-backend).
_This plugin was created through the Backstage CLI_
plugins/scaffolder-backend-module-gcp/api-report.md
+11
View File
@@ -0,0 +1,11 @@
## API Report File for "@backstage/plugin-scaffolder-backend-module-gcp"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { BackendFeatureCompat } from '@backstage/backend-plugin-api';
// @public
const gcpBucketModule: BackendFeatureCompat;
export default gcpBucketModule;
```
plugins/scaffolder-backend-module-gcp/catalog-info.yaml
+10
View File
@@ -0,0 +1,10 @@
apiVersion: backstage.io/v1alpha1
kind: Component
metadata:
name: backstage-plugin-scaffolder-backend-module-gcp
title: '@backstage/plugin-scaffolder-backend-module-gcp'
description: The GCP module for @backstage/plugin-scaffolder-backend
spec:
lifecycle: experimental
type: backstage-backend-plugin-module
owner: maintainers
plugins/scaffolder-backend-module-gcp/config.d.ts
Vendored
+25
View File
@@ -0,0 +1,25 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface Config {
/** Configuration options for the scaffolder plugin */
scaffolder?: {
/**
* Sets GCP bucket name to store serialized workspace for scaffolder tasks.
*/
EXPERIMENTAL_workspaceSerializationGcpBucketName?: string;
};
}
plugins/scaffolder-backend-module-gcp/knip-report.md
+2
View File
@@ -0,0 +1,2 @@
# Knip report
plugins/scaffolder-backend-module-gcp/package.json
+59
View File
@@ -0,0 +1,59 @@
{
"name": "@backstage/plugin-scaffolder-backend-module-gcp",
"version": "0.0.1-next.1",
"description": "The GCP Bucket module for @backstage/plugin-scaffolder-backend",
"backstage": {
"role": "backend-plugin-module",
"pluginId": "scaffolder",
"pluginPackage": "@backstage/plugin-scaffolder-backend"
},
"publishConfig": {
"access": "public"
},
"repository": {
"type": "git",
"url": "https://github.com/backstage/backstage",
"directory": "plugins/scaffolder-backend-module-gcp"
},
"license": "Apache-2.0",
"exports": {
".": "./src/index.ts",
"./package.json": "./package.json"
},
"main": "src/index.ts",
"types": "src/index.ts",
"typesVersions": {
"*": {
"package.json": [
"package.json"
]
}
},
"files": [
"dist",
"config.d.ts"
],
"scripts": {
"build": "backstage-cli package build",
"clean": "backstage-cli package clean",
"lint": "backstage-cli package lint",
"prepack": "backstage-cli package prepack",
"postpack": "backstage-cli package postpack",
"start": "backstage-cli package start",
"test": "backstage-cli package test"
},
"dependencies": {
"@backstage/backend-plugin-api": "workspace:^",
"@backstage/config": "workspace:^",
"@backstage/errors": "workspace:^",
"@backstage/integration": "workspace:^",
"@backstage/plugin-scaffolder-node": "workspace:^",
"@google-cloud/storage": "^7.0.0",
"raw-body": "^2.4.1"
},
"devDependencies": {
"@backstage/cli": "workspace:^",
"@backstage/plugin-scaffolder-node-test-utils": "workspace:^"
},
"configSchema": "config.d.ts"
}
plugins/scaffolder-backend-module-gcp/src/index.ts
+23
View File
@@ -0,0 +1,23 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* A module for the scaffolder backend that lets you interact with azure
*
* @packageDocumentation
*/
export { gcpBucketModule as default } from './module';
plugins/scaffolder-backend-module-gcp/src/module.ts
+44
View File
@@ -0,0 +1,44 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
coreServices,
createBackendModule,
} from '@backstage/backend-plugin-api';
import { scaffolderWorkspaceProviderExtensionPoint } from '@backstage/plugin-scaffolder-node/alpha';
import { GcpBucketWorkspaceProvider } from './providers/GcpBucketWorkspaceProvider';
/**
* @public
* The Azure Module for the Scaffolder Backend
*/
export const gcpBucketModule = createBackendModule({
moduleId: 'gcp',
pluginId: 'scaffolder',
register({ registerInit }) {
registerInit({
deps: {
scaffolderWorkspaceProviders: scaffolderWorkspaceProviderExtensionPoint,
config: coreServices.rootConfig,
logger: coreServices.logger,
},
async init({ config, logger, scaffolderWorkspaceProviders }) {
scaffolderWorkspaceProviders.addProviders({
gcpBucket: GcpBucketWorkspaceProvider.create(logger, config),
});
},
});
},
});
plugins/scaffolder-backend-module-gcp/src/providers/GcpBucketWorkspaceProvider.ts
+99
View File
@@ -0,0 +1,99 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Config } from '@backstage/config';
import { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha';
import getRawBody from 'raw-body';
import { Storage } from '@google-cloud/storage';
import { LoggerService } from '@backstage/backend-plugin-api';
import {
serializeWorkspace,
restoreWorkspace,
} from '@backstage/plugin-scaffolder-node/alpha';
export class GcpBucketWorkspaceProvider implements WorkspaceProvider {
static create(logger: LoggerService, config?: Config) {
return new GcpBucketWorkspaceProvider(new Storage(), logger, config);
}
private constructor(
private readonly storage: Storage,
private readonly logger: LoggerService,
private readonly config?: Config,
) {}
public async cleanWorkspace(options: { taskId: string }): Promise<void> {
const file = this.storage
.bucket(this.getGcpBucketName())
.file(options.taskId);
const result = await file.exists();
if (result[0]) {
await file.delete();
}
}
public async serializeWorkspace(options: {
path: string;
taskId: string;
}): Promise<void> {
const fileCloud = this.storage
.bucket(this.getGcpBucketName())
.file(options.taskId);
const { contents: workspace } = await serializeWorkspace(options);
try {
await fileCloud.save(workspace, {
contentType: 'application/x-tar',
});
} catch (err) {
this.logger.error(
`An error occurred during uploading the workspace of task ${
options.taskId
} into GCP bucket ${this.getGcpBucketName()}`,
);
}
this.logger.info(
`Workspace for task ${options.taskId} has been serialized.`,
);
}
public async rehydrateWorkspace(options: {
taskId: string;
targetPath: string;
}): Promise<void> {
const bucket = this.storage.bucket(this.getGcpBucketName());
const file = bucket.file(options.taskId);
const result = await file.exists();
if (result[0]) {
const workspace = await getRawBody(file.createReadStream());
await restoreWorkspace({ path: options.targetPath, buffer: workspace });
}
}
private getGcpBucketName(): string {
const bucketName = this.config?.getOptionalString(
'scaffolder.EXPERIMENTAL_workspaceSerializationGcpBucketName',
);
if (!bucketName) {
throw new Error(
`You've missed to configure scaffolder.EXPERIMENTAL_workspaceSerializationGcpBucketName in app-config.yaml file`,
);
}
return bucketName;
}
}
plugins/scaffolder-backend/api-report.md
+4
View File
@@ -64,6 +64,7 @@ import { TemplateFilter as TemplateFilter_2 } from '@backstage/plugin-scaffolder
import { TemplateGlobal as TemplateGlobal_2 } from '@backstage/plugin-scaffolder-node';
import { TemplateParametersV1beta3 } from '@backstage/plugin-scaffolder-common';
import { UrlReader } from '@backstage/backend-common';
import { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha';
import { ZodType } from 'zod';
import { ZodTypeDef } from 'zod';
@@ -477,6 +478,8 @@ export interface RouterOptions {
// (undocumented)
additionalTemplateGlobals?: Record<string, TemplateGlobal_2>;
// (undocumented)
additionalWorkspaceProviders?: Record<string, WorkspaceProvider>;
// (undocumented)
auth?: AuthService;
// (undocumented)
autocompleteHandlers?: Record<string, AutocompleteHandler>;
@@ -559,6 +562,7 @@ export class TaskManager implements TaskContext_2 {
logger: Logger,
auth?: AuthService,
config?: Config,
additionalWorkspaceProviders?: Record<string, WorkspaceProvider>,
): TaskManager;
// (undocumented)
get createdBy(): string | undefined;
plugins/scaffolder-backend/config.d.ts
Vendored
+7
View File
@@ -52,6 +52,13 @@ export interface Config {
*/
EXPERIMENTAL_workspaceSerialization?: boolean;
/**
* Sets the provider for workspace serialization.
*
* By default, it is your database.
*/
EXPERIMENTAL_workspaceSerializationProvider?: string;
/**
* Every task which is in progress state and having a last heartbeat longer than a specified timeout is going to
* be attempted to recover.
plugins/scaffolder-backend/src/ScaffolderPlugin.ts
+10
View File
@@ -33,6 +33,8 @@ import {
scaffolderAutocompleteExtensionPoint,
scaffolderTaskBrokerExtensionPoint,
scaffolderTemplatingExtensionPoint,
scaffolderWorkspaceProviderExtensionPoint,
WorkspaceProvider,
} from '@backstage/plugin-scaffolder-node/alpha';
import {
createCatalogRegisterAction,
@@ -91,6 +93,13 @@ export const scaffolderPlugin = createBackendPlugin({
},
});
const additionalWorkspaceProviders: Record<string, WorkspaceProvider> = {};
env.registerExtensionPoint(scaffolderWorkspaceProviderExtensionPoint, {
addProviders(provider) {
Object.assign(additionalWorkspaceProviders, provider);
},
});
env.registerInit({
deps: {
logger: coreServices.logger,
@@ -172,6 +181,7 @@ export const scaffolderPlugin = createBackendPlugin({
discovery,
permissions,
autocompleteHandlers,
additionalWorkspaceProviders,
});
httpRouter.use(router);
},
plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts
+9 -3
View File
@@ -40,7 +40,10 @@ import { DateTime, Duration } from 'luxon';
import { TaskRecovery, TaskSpec } from '@backstage/plugin-scaffolder-common';
import { trimEventsTillLastRecovery } from './taskRecoveryHelper';
import { intervalFromNowTill } from './dbUtil';
import { restoreWorkspace, serializeWorkspace } from './serializer';
import {
restoreWorkspace,
serializeWorkspace,
} from '@backstage/plugin-scaffolder-node/alpha';
const migrationsDir = resolvePackagePath(
'@backstage/plugin-scaffolder-backend',
@@ -517,7 +520,10 @@ export class DatabaseTaskStore implements TaskStore {
.where({ id: options.taskId })
.select('workspace');
await restoreWorkspace(options.targetPath, result.workspace);
await restoreWorkspace({
path: options.targetPath,
buffer: result.workspace,
});
}
async cleanWorkspace({ taskId }: { taskId: string }): Promise<void> {
@@ -534,7 +540,7 @@ export class DatabaseTaskStore implements TaskStore {
await this.db<RawDbTaskRow>('tasks')
.where({ id: options.taskId })
.update({
workspace: await serializeWorkspace(options.path),
workspace: (await serializeWorkspace(options)).contents,
});
}
}
plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseWorkspaceProvider.ts
+45
View File
@@ -0,0 +1,45 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TaskStore } from './types';
import { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha';
export class DatabaseWorkspaceProvider implements WorkspaceProvider {
static create(storage: TaskStore) {
return new DatabaseWorkspaceProvider(storage);
}
private constructor(private readonly storage: TaskStore) {}
public async serializeWorkspace(options: {
path: string;
taskId: string;
}): Promise<void> {
this.storage.serializeWorkspace?.(options);
}
public async rehydrateWorkspace(options: {
taskId: string;
targetPath: string;
}): Promise<void> {
return this.storage.rehydrateWorkspace?.(options);
}
public async cleanWorkspace(options: { taskId: string }): Promise<void> {
return this.storage.cleanWorkspace?.(options);
}
}
plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
+1 -1
View File
@@ -443,7 +443,6 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
throw new Error(`Step ${step.name} has been cancelled.`);
}
await task.cleanWorkspace?.();
await stepTrack.markSuccessful();
} catch (err) {
await taskTrack.markFailed(step, err);
@@ -513,6 +512,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
return { output };
} finally {
if (workspacePath) {
await task.cleanWorkspace?.();
await fs.remove(workspacePath);
}
}
plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts
+20 -22
View File
@@ -34,6 +34,8 @@ import {
AuthService,
BackstageCredentials,
} from '@backstage/backend-plugin-api';
import { DefaultWorkspaceService, WorkspaceService } from './WorkspaceService';
import { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha';
type TaskState = {
checkpoints: {
@@ -65,14 +67,22 @@ export class TaskManager implements TaskContext {
logger: Logger,
auth?: AuthService,
config?: Config,
additionalWorkspaceProviders?: Record<string, WorkspaceProvider>,
) {
const workspaceService = DefaultWorkspaceService.create(
task,
storage,
additionalWorkspaceProviders,
config,
);
const agent = new TaskManager(
task,
storage,
abortSignal,
logger,
workspaceService,
auth,
config,
);
agent.startTimeout();
return agent;
@@ -84,8 +94,8 @@ export class TaskManager implements TaskContext {
private readonly storage: TaskStore,
private readonly signal: AbortSignal,
private readonly logger: Logger,
private readonly workspaceService: WorkspaceService,
private readonly auth?: AuthService,
private readonly config?: Config,
) {}
get spec() {
@@ -112,9 +122,7 @@ export class TaskManager implements TaskContext {
taskId: string;
targetPath: string;
}): Promise<void> {
if (this.isWorkspaceSerializationEnabled()) {
this.storage.rehydrateWorkspace?.(options);
}
await this.workspaceService.rehydrateWorkspace(options);
}
get done() {
@@ -163,18 +171,11 @@ export class TaskManager implements TaskContext {
}
async serializeWorkspace?(options: { path: string }): Promise<void> {
if (this.isWorkspaceSerializationEnabled()) {
await this.storage.serializeWorkspace?.({
path: options.path,
taskId: this.task.taskId,
});
}
await this.workspaceService.serializeWorkspace(options);
}
async cleanWorkspace?(): Promise<void> {
if (this.isWorkspaceSerializationEnabled()) {
await this.storage.cleanWorkspace?.({ taskId: this.task.taskId });
}
await this.workspaceService.cleanWorkspace();
}
async complete(
@@ -211,14 +212,6 @@ export class TaskManager implements TaskContext {
}, 1000);
}
private isWorkspaceSerializationEnabled(): boolean {
return (
this.config?.getOptionalBoolean(
'scaffolder.EXPERIMENTAL_workspaceSerialization',
) ?? false
);
}
async getInitiatorCredentials(): Promise<BackstageCredentials> {
const secrets = this.task.secrets as InternalTaskSecrets;
@@ -278,6 +271,10 @@ export class StorageTaskBroker implements TaskBroker {
private readonly logger: Logger,
private readonly config?: Config,
private readonly auth?: AuthService,
private readonly additionalWorkspaceProviders?: Record<
string,
WorkspaceProvider
>,
) {}
async list(options?: {
@@ -366,6 +363,7 @@ export class StorageTaskBroker implements TaskBroker {
this.logger,
this.auth,
this.config,
this.additionalWorkspaceProviders,
);
}
plugins/scaffolder-backend/src/scaffolder/tasks/WorkspaceService.ts
+88
View File
@@ -0,0 +1,88 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Config } from '@backstage/config';
import { CurrentClaimedTask } from './StorageTaskBroker';
import { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha';
import { DatabaseWorkspaceProvider } from './DatabaseWorkspaceProvider';
import { TaskStore } from './types';
export interface WorkspaceService {
serializeWorkspace(options: { path: string }): Promise<void>;
cleanWorkspace(): Promise<void>;
rehydrateWorkspace(options: {
taskId: string;
targetPath: string;
}): Promise<void>;
}
export class DefaultWorkspaceService implements WorkspaceService {
static create(
task: CurrentClaimedTask,
storage: TaskStore,
additionalWorkspaceProviders?: Record<string, WorkspaceProvider>,
config?: Config,
) {
const workspaceProviderName =
config?.getOptionalString(
'scaffolder.EXPERIMENTAL_workspaceSerializationProvider',
) ?? 'database';
const workspaceProvider =
additionalWorkspaceProviders?.[workspaceProviderName] ??
DatabaseWorkspaceProvider.create(storage);
return new DefaultWorkspaceService(task, workspaceProvider, config);
}
private constructor(
private readonly task: CurrentClaimedTask,
private readonly workspaceProvider: WorkspaceProvider,
private readonly config?: Config,
) {}
public async serializeWorkspace(options: { path: string }): Promise<void> {
if (this.isWorkspaceSerializationEnabled()) {
await this.workspaceProvider.serializeWorkspace({
path: options.path,
taskId: this.task.taskId,
});
}
}
public async cleanWorkspace(): Promise<void> {
if (this.isWorkspaceSerializationEnabled()) {
await this.workspaceProvider.cleanWorkspace({ taskId: this.task.taskId });
}
}
public async rehydrateWorkspace(options: {
taskId: string;
targetPath: string;
}): Promise<void> {
if (this.isWorkspaceSerializationEnabled()) {
await this.workspaceProvider.rehydrateWorkspace(options);
}
}
private isWorkspaceSerializationEnabled(): boolean {
return (
this.config?.getOptionalBoolean(
'scaffolder.EXPERIMENTAL_workspaceSerialization',
) ?? false
);
}
}
plugins/scaffolder-backend/src/service/router.ts
+13 -2
View File
@@ -94,7 +94,10 @@ import {
} from '@backstage/plugin-auth-node';
import { InternalTaskSecrets } from '../scaffolder/tasks/types';
import { checkPermission } from '../util/checkPermissions';
import { AutocompleteHandler } from '@backstage/plugin-scaffolder-node/alpha';
import {
AutocompleteHandler,
WorkspaceProvider,
} from '@backstage/plugin-scaffolder-node/alpha';
/**
*
@@ -159,6 +162,7 @@ export interface RouterOptions {
taskBroker?: TaskBroker;
additionalTemplateFilters?: Record<string, TemplateFilter>;
additionalTemplateGlobals?: Record<string, TemplateGlobal>;
additionalWorkspaceProviders?: Record<string, WorkspaceProvider>;
permissions?: PermissionsService;
permissionRules?: Array<
TemplatePermissionRuleInput | ActionPermissionRuleInput
@@ -272,6 +276,7 @@ export async function createRouter(
scheduler,
additionalTemplateFilters,
additionalTemplateGlobals,
additionalWorkspaceProviders,
permissions,
permissionRules,
discovery = HostDiscovery.fromConfig(config),
@@ -297,7 +302,13 @@ export async function createRouter(
let taskBroker: TaskBroker;
if (!options.taskBroker) {
const databaseTaskStore = await DatabaseTaskStore.create({ database });
taskBroker = new StorageTaskBroker(databaseTaskStore, logger, config, auth);
taskBroker = new StorageTaskBroker(
databaseTaskStore,
logger,
config,
auth,
additionalWorkspaceProviders,
);
if (scheduler && databaseTaskStore.listStaleTasks) {
await scheduler.scheduleTask({
plugins/scaffolder-node/api-report-alpha.md
+41
View File
@@ -3,6 +3,8 @@
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
/// <reference types="node" />
import { ExtensionPoint } from '@backstage/backend-plugin-api';
import { TaskBroker } from '@backstage/plugin-scaffolder-node';
import { TemplateAction } from '@backstage/plugin-scaffolder-node';
@@ -24,6 +26,12 @@ export type AutocompleteHandler = ({
}[];
}>;
// @alpha
export const restoreWorkspace: (opts: {
path: string;
buffer?: Buffer;
}) => Promise<void>;
// @alpha
export interface ScaffolderActionsExtensionPoint {
// (undocumented)
@@ -68,5 +76,38 @@ export interface ScaffolderTemplatingExtensionPoint {
// @alpha
export const scaffolderTemplatingExtensionPoint: ExtensionPoint<ScaffolderTemplatingExtensionPoint>;
// @alpha
export interface ScaffolderWorkspaceProviderExtensionPoint {
// (undocumented)
addProviders(providers: Record<string, WorkspaceProvider>): void;
}
// @alpha
export const scaffolderWorkspaceProviderExtensionPoint: ExtensionPoint<ScaffolderWorkspaceProviderExtensionPoint>;
// @alpha
export const serializeWorkspace: (opts: { path: string }) => Promise<{
contents: Buffer;
}>;
// @alpha
export interface WorkspaceProvider {
// (undocumented)
cleanWorkspace(options: { taskId: string }): Promise<void>;
// (undocumented)
rehydrateWorkspace(options: {
taskId: string;
targetPath: string;
}): Promise<void>;
// (undocumented)
serializeWorkspace({
path,
taskId,
}: {
path: string;
taskId: string;
}): Promise<void>;
}
// (No @packageDocumentation comment for this package)
```
plugins/scaffolder-node/package.json
+2
View File
@@ -61,11 +61,13 @@
"@backstage/integration": "workspace:^",
"@backstage/plugin-scaffolder-common": "workspace:^",
"@backstage/types": "workspace:^",
"concat-stream": "^2.0.0",
"fs-extra": "^11.2.0",
"globby": "^11.0.0",
"isomorphic-git": "^1.23.0",
"jsonschema": "^1.2.6",
"p-limit": "^3.1.0",
"tar": "^6.1.12",
"winston": "^3.2.1",
"zod": "^3.22.4",
"zod-to-json-schema": "^3.20.4"
plugins/scaffolder-node/src/alpha.ts
+45 -1
View File
@@ -22,6 +22,8 @@ import {
TaskBroker,
} from '@backstage/plugin-scaffolder-node';
export * from './tasks/alpha';
/**
* Extension point for managing scaffolder actions.
*
@@ -67,6 +69,7 @@ export const scaffolderTaskBrokerExtensionPoint =
*/
export interface ScaffolderTemplatingExtensionPoint {
addTemplateFilters(filters: Record<string, TemplateFilter>): void;
addTemplateGlobals(filters: Record<string, TemplateGlobal>): void;
}
@@ -109,7 +112,7 @@ export interface ScaffolderAutocompleteExtensionPoint {
}
/**
* Extension point for adding template filters and globals.
* Extension point for adding autocomplete handlers.
*
* @alpha
*/
@@ -117,3 +120,44 @@ export const scaffolderAutocompleteExtensionPoint =
createExtensionPoint<ScaffolderAutocompleteExtensionPoint>({
id: 'scaffolder.autocomplete',
});
/**
* This provider has to be implemented to make it possible to serialize/deserialize scaffolder workspace.
*
* @alpha
*/
export interface WorkspaceProvider {
serializeWorkspace({
path,
taskId,
}: {
path: string;
taskId: string;
}): Promise<void>;
cleanWorkspace(options: { taskId: string }): Promise<void>;
rehydrateWorkspace(options: {
taskId: string;
targetPath: string;
}): Promise<void>;
}
/**
* Extension point for adding workspace providers.
*
* @alpha
*/
export interface ScaffolderWorkspaceProviderExtensionPoint {
addProviders(providers: Record<string, WorkspaceProvider>): void;
}
/**
* Extension point for adding workspace providers.
*
* @alpha
*/
export const scaffolderWorkspaceProviderExtensionPoint =
createExtensionPoint<ScaffolderWorkspaceProviderExtensionPoint>({
id: 'scaffolder.workspace.provider',
});
plugins/scaffolder-node/src/tasks/alpha.ts
+16
View File
@@ -0,0 +1,16 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export * from './serializer';
plugins/scaffolder-backend/src/scaffolder/tasks/serializer.test.ts → plugins/scaffolder-node/src/tasks/serializer.test.ts
+5 -2
View File
@@ -72,8 +72,11 @@ describe('serializer', () => {
const restoredWorkspaceDir = createMockDirectory();
it('should be able to archive and restore the workspace', async () => {
const workspaceBuffer = await serializeWorkspace(workspaceDir.path);
await restoreWorkspace(restoredWorkspaceDir.path, workspaceBuffer);
const workspaceBuffer = await serializeWorkspace(workspaceDir);
await restoreWorkspace({
path: restoredWorkspaceDir.path,
buffer: workspaceBuffer.contents,
});
expect(
fs.existsSync(`${restoredWorkspaceDir.path}/\$\{ESCAPE_ME\}.txt`),
plugins/scaffolder-backend/src/scaffolder/tasks/serializer.ts → plugins/scaffolder-node/src/tasks/serializer.ts
+25 -5
View File
@@ -20,14 +20,34 @@ import { promisify } from 'util';
import { pipeline as pipelineCb, Readable } from 'stream';
const pipeline = promisify(pipelineCb);
export const serializeWorkspace = async (path: string): Promise<Buffer> => {
return await new Promise<Buffer>(async resolve => {
await pipeline(tar.create({ cwd: path }, ['']), concatStream(resolve));
/**
* Serializes provided path into tar archive
*
* @alpha
*/
export const serializeWorkspace = async (opts: {
path: string;
}): Promise<{ contents: Buffer }> => {
return new Promise<{ contents: Buffer }>(async resolve => {
await pipeline(
tar.create({ cwd: opts.path }, ['']),
concatStream(buffer => {
return resolve({ contents: buffer });
}),
);
});
};
export const restoreWorkspace = async (path: string, buffer?: Buffer) => {
/**
* Rehydrates the provided buffer of tar archive into the provide destination path
*
* @alpha
*/
export const restoreWorkspace = async (opts: {
path: string;
buffer?: Buffer;
}): Promise<void> => {
const { buffer, path } = opts;
if (buffer) {
await pipeline(
Readable.from(buffer),
yarn.lock
+26 -15
View File
@@ -6779,6 +6779,22 @@ __metadata:
languageName: unknown
linkType: soft
"@backstage/plugin-scaffolder-backend-module-gcp@workspace:plugins/scaffolder-backend-module-gcp":
version: 0.0.0-use.local
resolution: "@backstage/plugin-scaffolder-backend-module-gcp@workspace:plugins/scaffolder-backend-module-gcp"
dependencies:
"@backstage/backend-plugin-api": "workspace:^"
"@backstage/cli": "workspace:^"
"@backstage/config": "workspace:^"
"@backstage/errors": "workspace:^"
"@backstage/integration": "workspace:^"
"@backstage/plugin-scaffolder-node": "workspace:^"
"@backstage/plugin-scaffolder-node-test-utils": "workspace:^"
"@google-cloud/storage": ^7.0.0
raw-body: ^2.4.1
languageName: unknown
linkType: soft
"@backstage/plugin-scaffolder-backend-module-gerrit@workspace:^, @backstage/plugin-scaffolder-backend-module-gerrit@workspace:plugins/scaffolder-backend-module-gerrit":
version: 0.0.0-use.local
resolution: "@backstage/plugin-scaffolder-backend-module-gerrit@workspace:plugins/scaffolder-backend-module-gerrit"
@@ -7042,11 +7058,13 @@ __metadata:
"@backstage/integration": "workspace:^"
"@backstage/plugin-scaffolder-common": "workspace:^"
"@backstage/types": "workspace:^"
concat-stream: ^2.0.0
fs-extra: ^11.2.0
globby: ^11.0.0
isomorphic-git: ^1.23.0
jsonschema: ^1.2.6
p-limit: ^3.1.0
tar: ^6.1.12
winston: ^3.2.1
zod: ^3.22.4
zod-to-json-schema: ^3.20.4
@@ -9355,8 +9373,8 @@ __metadata:
linkType: hard
"@google-cloud/storage@npm:^7.0.0":
version: 7.10.1
resolution: "@google-cloud/storage@npm:7.10.1"
version: 7.11.2
resolution: "@google-cloud/storage@npm:7.11.2"
dependencies:
"@google-cloud/paginator": ^5.0.0
"@google-cloud/projectify": ^4.0.0
@@ -9364,16 +9382,16 @@ __metadata:
abort-controller: ^3.0.0
async-retry: ^1.3.3
duplexify: ^4.1.3
ent: ^2.2.0
fast-xml-parser: ^4.3.0
gaxios: ^6.0.2
google-auth-library: ^9.6.3
html-entities: ^2.5.2
mime: ^3.0.0
p-limit: ^3.0.1
retry-request: ^7.0.0
teeny-request: ^9.0.0
uuid: ^8.0.0
checksum: 0605f8abf67cde98bc48b05c749b569416af821fcc3eb8ed3fac6dc5ade9c1f2f3210404c85e30525260ea4f34785d7436f7684e6eacd083613237669badff25
checksum: 763cd560245006e11a060fdbd5edff5d6bfe22b0c0b06dfe88af43b7b14458598c00a71c4e1b079e9ab09096d7d262bdc27fa77d76a8969af0765f23c93f8405
languageName: node
linkType: hard
@@ -25281,13 +25299,6 @@ __metadata:
languageName: node
linkType: hard
"ent@npm:^2.2.0":
version: 2.2.0
resolution: "ent@npm:2.2.0"
checksum: f588b5707d6fef36011ea10d530645912a69530a1eb0831f8708c498ac028363a7009f45cfadd28ceb4dafd9ac17ec15213f88d09ce239cd033cfe1328dd7d7d
languageName: node
linkType: hard
"entities@npm:^2.0.0":
version: 2.2.0
resolution: "entities@npm:2.2.0"
@@ -28821,10 +28832,10 @@ __metadata:
languageName: node
linkType: hard
"html-entities@npm:^2.1.0, html-entities@npm:^2.4.0":
version: 2.4.0
resolution: "html-entities@npm:2.4.0"
checksum: 25bea32642ce9ebd0eedc4d24381883ecb0335ccb8ac26379a0958b9b16652fdbaa725d70207ce54a51db24103436a698a8e454397d3ba8ad81460224751f1dc
"html-entities@npm:^2.1.0, html-entities@npm:^2.4.0, html-entities@npm:^2.5.2":
version: 2.5.2
resolution: "html-entities@npm:2.5.2"
checksum: b23f4a07d33d49ade1994069af4e13d31650e3fb62621e92ae10ecdf01d1a98065c78fd20fdc92b4c7881612210b37c275f2c9fba9777650ab0d6f2ceb3b99b6
languageName: node
linkType: hard