From 2b900fee6dcd956bff1503bf89a602d45a7a195e Mon Sep 17 00:00:00 2001 From: bnechyporenko Date: Sat, 10 Feb 2024 14:34:14 +0100 Subject: [PATCH] wip Signed-off-by: bnechyporenko --- plugins/scaffolder-backend/api-report.md | 123 ++++++++++++++---- .../src/scaffolder/tasks/DatabaseTaskStore.ts | 51 ++++++-- .../tasks/NunjucksWorkflowRunner.ts | 2 +- .../src/scaffolder/tasks/StorageTaskBroker.ts | 47 +++++-- .../src/scaffolder/tasks/index.ts | 1 - .../src/scaffolder/tasks/types.ts | 42 +++--- plugins/scaffolder-node/api-report.md | 53 +++++--- plugins/scaffolder-node/src/tasks/index.ts | 1 - plugins/scaffolder-node/src/tasks/types.ts | 55 +++++--- 9 files changed, 274 insertions(+), 101 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 742834a4df..b226372d86 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -9,7 +9,6 @@ import * as bitbucket from '@backstage/plugin-scaffolder-backend-module-bitbucke import * as bitbucketCloud from '@backstage/plugin-scaffolder-backend-module-bitbucket-cloud'; import * as bitbucketServer from '@backstage/plugin-scaffolder-backend-module-bitbucket-server'; import { CatalogApi } from '@backstage/catalog-client'; -import { CheckpointRecord } from '@backstage/plugin-scaffolder-node'; import { Config } from '@backstage/config'; import { Duration } from 'luxon'; import { executeShellCommand as executeShellCommand_2 } from '@backstage/plugin-scaffolder-node'; @@ -22,6 +21,7 @@ import * as gitlab from '@backstage/plugin-scaffolder-backend-module-gitlab'; import { HumanDuration } from '@backstage/types'; import { IdentityApi } from '@backstage/plugin-auth-node'; import { JsonObject } from '@backstage/types'; +import { JsonValue } from '@backstage/types'; import { Knex } from 'knex'; import { LifecycleService } from '@backstage/backend-plugin-api'; import { Logger } from 'winston'; @@ -48,7 +48,6 @@ import { TaskRecovery } from '@backstage/plugin-scaffolder-common'; import { TaskSecrets as TaskSecrets_2 } from '@backstage/plugin-scaffolder-node'; import { TaskSpec } from '@backstage/plugin-scaffolder-common'; import { TaskSpecV1beta3 } from '@backstage/plugin-scaffolder-common'; -import { TaskState } from '@backstage/plugin-scaffolder-node'; import { TaskStatus as TaskStatus_2 } from '@backstage/plugin-scaffolder-node'; import { TemplateAction as TemplateAction_2 } from '@backstage/plugin-scaffolder-node'; import { TemplateActionOptions } from '@backstage/plugin-scaffolder-node'; @@ -361,7 +360,17 @@ export interface CurrentClaimedTask { createdBy?: string; secrets?: TaskSecrets_2; spec: TaskSpec; - state?: TaskState; + state?: { + [key: string]: + | { + status: 'failed'; + reason: string; + } + | { + status: 'success'; + value: JsonValue; + }; + }; taskId: string; } @@ -400,16 +409,29 @@ export class DatabaseTaskStore implements TaskStore { // (undocumented) getTask(taskId: string): Promise; // (undocumented) + getTaskState({ taskId }: { taskId: string }): Promise< + | { + state: { + [key: string]: + | { + status: 'failed'; + reason: string; + } + | { + status: 'success'; + value: JsonValue; + }; + }; + } + | undefined + >; + // (undocumented) heartbeatTask(taskId: string): Promise; // (undocumented) list(options: { createdBy?: string }): Promise<{ tasks: SerializedTask_2[]; }>; // (undocumented) - listCheckpoints({ taskId }: { taskId: string }): Promise<{ - state: TaskState; - }>; - // (undocumented) listEvents(options: TaskStoreListEventsOptions): Promise<{ events: SerializedTaskEvent_2[]; }>; @@ -425,7 +447,22 @@ export class DatabaseTaskStore implements TaskStore { ids: string[]; }>; // (undocumented) - saveCheckpoint(options: TaskStoreStateOptions): Promise; + saveCheckpoint(options: { + taskId: string; + state?: + | { + [key: string]: + | { + status: 'failed'; + reason: string; + } + | { + status: 'success'; + value: JsonValue; + }; + } + | undefined; + }): Promise; // (undocumented) shutdownTask(options: TaskStoreShutDownTaskOptions): Promise; } @@ -528,9 +565,19 @@ export class TaskManager implements TaskContext { // (undocumented) emitLog(message: string, logMetadata?: JsonObject): Promise; // (undocumented) - getCheckpoints?(): Promise< + getTaskState?(): Promise< | { - state: TaskState; + state: { + [key: string]: + | { + status: 'failed'; + reason: string; + } + | { + status: 'success'; + value: JsonValue; + }; + }; } | undefined >; @@ -541,7 +588,19 @@ export class TaskManager implements TaskContext { // (undocumented) get spec(): TaskSpecV1beta3; // (undocumented) - updateCheckpoint?(options: CheckpointRecord): Promise; + updateCheckpoint?( + options: + | { + key: string; + status: 'success'; + value: JsonValue; + } + | { + key: string; + status: 'failed'; + reason: string; + }, + ): Promise; } // @public @deprecated (undocumented) @@ -571,16 +630,29 @@ export interface TaskStore { // (undocumented) getTask(taskId: string): Promise; // (undocumented) + getTaskState?({ taskId }: { taskId: string }): Promise< + | { + state: { + [key: string]: + | { + status: 'failed'; + reason: string; + } + | { + status: 'success'; + value: JsonValue; + }; + }; + } + | undefined + >; + // (undocumented) heartbeatTask(taskId: string): Promise; // (undocumented) list?(options: { createdBy?: string }): Promise<{ tasks: SerializedTask[]; }>; // (undocumented) - listCheckpoints?({ taskId }: { taskId: string }): Promise<{ - state: TaskState; - }>; - // (undocumented) listEvents(options: TaskStoreListEventsOptions): Promise<{ events: SerializedTaskEvent[]; }>; @@ -595,7 +667,20 @@ export interface TaskStore { ids: string[]; }>; // (undocumented) - saveCheckpoint?(options: TaskStoreStateOptions): Promise; + saveCheckpoint?(options: { + taskId: string; + state?: { + [key: string]: + | { + status: 'failed'; + reason: string; + } + | { + status: 'success'; + value: JsonValue; + }; + }; + }): Promise; // (undocumented) shutdownTask?(options: TaskStoreShutDownTaskOptions): Promise; } @@ -634,12 +719,6 @@ export type TaskStoreShutDownTaskOptions = { taskId: string; }; -// @public -export type TaskStoreStateOptions = { - taskId: string; - state?: TaskState; -}; - // @public export class TaskWorker { // (undocumented) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 14535d034c..0d6521dfd2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { JsonObject } from '@backstage/types'; +import { JsonObject, JsonValue } from '@backstage/types'; import { PluginDatabaseManager, resolvePackagePath, @@ -30,7 +30,6 @@ import { TaskStoreCreateTaskResult, TaskStoreShutDownTaskOptions, TaskStoreRecoverTaskOptions, - TaskStoreStateOptions, } from './types'; import { SerializedTaskEvent, @@ -38,7 +37,6 @@ import { TaskStatus, TaskEventType, TaskSecrets, - TaskState, } from '@backstage/plugin-scaffolder-node'; import { DateTime, Duration } from 'luxon'; import { TaskRecovery, TaskSpec } from '@backstage/plugin-scaffolder-common'; @@ -397,18 +395,49 @@ export class DatabaseTaskStore implements TaskStore { }); } - async listCheckpoints({ - taskId, - }: { - taskId: string; - }): Promise<{ state: TaskState }> { - const state = await this.db('tasks') + async getTaskState({ taskId }: { taskId: string }): Promise< + | { + state: { + [key: string]: + | { status: 'failed'; reason: string } + | { + status: 'success'; + value: JsonValue; + }; + }; + } + | undefined + > { + const [result] = await this.db('tasks') .where({ id: taskId }) .select('state'); - return { state: JSON.stringify(state) as unknown as TaskState }; + return result.state + ? { + state: JSON.parse(result.state) as unknown as { + [key: string]: + | { status: 'failed'; reason: string } + | { + status: 'success'; + value: JsonValue; + }; + }, + } + : undefined; } - async saveCheckpoint(options: TaskStoreStateOptions): Promise { + async saveCheckpoint(options: { + taskId: string; + state?: + | { + [key: string]: + | { status: 'failed'; reason: string } + | { + status: 'success'; + value: JsonValue; + }; + } + | undefined; + }): Promise { if (options.state) { const serializedState = JSON.stringify(options.state); await this.db('tasks') diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index d9c1979eee..b5ffc73189 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -332,7 +332,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { } const tmpDirs = new Array(); const stepOutput: { [outputName: string]: JsonValue } = {}; - const prevTaskState = await task.getCheckpoints?.(); + const prevTaskState = await task.getTaskState?.(); for (const iteration of iterations) { if (iteration.each) { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 6e01141dab..453df09c77 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -16,12 +16,8 @@ import { Config } from '@backstage/config'; import { TaskSpec } from '@backstage/plugin-scaffolder-common'; -import { - TaskSecrets, - TaskState, - CheckpointRecord, -} from '@backstage/plugin-scaffolder-node'; -import { JsonObject, Observable } from '@backstage/types'; +import { TaskSecrets } from '@backstage/plugin-scaffolder-node'; +import { JsonObject, JsonValue, Observable } from '@backstage/types'; import { Logger } from 'winston'; import ObservableImpl from 'zen-observable'; import { @@ -95,11 +91,35 @@ export class TaskManager implements TaskContext { }); } - async getCheckpoints?(): Promise<{ state: TaskState } | undefined> { - return this.storage.listCheckpoints?.({ taskId: this.task.taskId }); + async getTaskState?(): Promise< + | { + state: { + [key: string]: + | { status: 'failed'; reason: string } + | { + status: 'success'; + value: JsonValue; + }; + }; + } + | undefined + > { + return this.storage.getTaskState?.({ taskId: this.task.taskId }); } - async updateCheckpoint?(options: CheckpointRecord): Promise { + async updateCheckpoint?( + options: + | { + key: string; + status: 'success'; + value: JsonValue; + } + | { + key: string; + status: 'failed'; + reason: string; + }, + ): Promise { const { key, ...value } = options; if (this.task.state) { this.task.state[key] = value; @@ -168,7 +188,14 @@ export interface CurrentClaimedTask { /** * The state of checkpoints of the task. */ - state?: TaskState; + state?: { + [key: string]: + | { status: 'failed'; reason: string } + | { + status: 'success'; + value: JsonValue; + }; + }; /** * The creator of the task. */ diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index 2da812e869..9d231d7e7c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -25,7 +25,6 @@ export type { TaskStoreEmitOptions, TaskStoreListEventsOptions, TaskStoreShutDownTaskOptions, - TaskStoreStateOptions, SerializedTask, SerializedTaskEvent, TaskStatus, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index e30de5db94..215defd280 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -16,7 +16,7 @@ import { JsonValue, JsonObject, HumanDuration } from '@backstage/types'; import { TaskSpec, TaskStep } from '@backstage/plugin-scaffolder-common'; -import { TaskSecrets, TaskState } from '@backstage/plugin-scaffolder-node'; +import { TaskSecrets } from '@backstage/plugin-scaffolder-node'; import { TemplateAction, TaskStatus as _TaskStatus, @@ -113,16 +113,6 @@ export type TaskStoreEmitOptions = { body: TBody; }; -/** - * TaskStoreStateOptions - * - * @public - */ -export type TaskStoreStateOptions = { - taskId: string; - state?: TaskState; -}; - /** * TaskStoreListEventsOptions * @@ -204,13 +194,31 @@ export interface TaskStore { emitLogEvent(options: TaskStoreEmitOptions): Promise; - listCheckpoints?({ - taskId, - }: { - taskId: string; - }): Promise<{ state: TaskState }>; + getTaskState?({ taskId }: { taskId: string }): Promise< + | { + state: { + [key: string]: + | { status: 'failed'; reason: string } + | { + status: 'success'; + value: JsonValue; + }; + }; + } + | undefined + >; - saveCheckpoint?(options: TaskStoreStateOptions): Promise; + saveCheckpoint?(options: { + taskId: string; + state?: { + [key: string]: + | { status: 'failed'; reason: string } + | { + status: 'success'; + value: JsonValue; + }; + }; + }): Promise; listEvents( options: TaskStoreListEventsOptions, diff --git a/plugins/scaffolder-node/api-report.md b/plugins/scaffolder-node/api-report.md index d3bc9d8500..c87ec8183c 100644 --- a/plugins/scaffolder-node/api-report.md +++ b/plugins/scaffolder-node/api-report.md @@ -64,19 +64,6 @@ export function addFiles(options: { logger?: Logger | undefined; }): Promise; -// @public -export type CheckpointRecord = - | { - key: string; - status: 'success'; - value: JsonValue; - } - | { - key: string; - status: 'failed'; - reason: string; - }; - // @public (undocumented) export function cloneRepo(options: { url: string; @@ -362,9 +349,19 @@ export interface TaskContext { // (undocumented) emitLog(message: string, logMetadata?: JsonObject): Promise; // (undocumented) - getCheckpoints?(): Promise< + getTaskState?(): Promise< | { - state: TaskState; + state: { + [key: string]: + | { + status: 'failed'; + reason: string; + } + | { + status: 'success'; + value: JsonValue; + }; + }; } | undefined >; @@ -377,9 +374,31 @@ export interface TaskContext { // (undocumented) spec: TaskSpec; // (undocumented) - state?: TaskState; + state?: { + [key: string]: + | { + status: 'failed'; + reason: string; + } + | { + status: 'success'; + value: JsonValue; + }; + }; // (undocumented) - updateCheckpoint?(options: CheckpointRecord): Promise; + updateCheckpoint?( + options: + | { + key: string; + status: 'success'; + value: JsonValue; + } + | { + key: string; + status: 'failed'; + reason: string; + }, + ): Promise; } // @public diff --git a/plugins/scaffolder-node/src/tasks/index.ts b/plugins/scaffolder-node/src/tasks/index.ts index 60023bf48e..99638e48af 100644 --- a/plugins/scaffolder-node/src/tasks/index.ts +++ b/plugins/scaffolder-node/src/tasks/index.ts @@ -26,5 +26,4 @@ export type { TaskEventType, TaskState, TaskStatus, - CheckpointRecord, } from './types'; diff --git a/plugins/scaffolder-node/src/tasks/types.ts b/plugins/scaffolder-node/src/tasks/types.ts index a7fbf4ac76..46b82ebe53 100644 --- a/plugins/scaffolder-node/src/tasks/types.ts +++ b/plugins/scaffolder-node/src/tasks/types.ts @@ -26,24 +26,6 @@ export type TaskSecrets = Record & { backstageToken?: string; }; -/** - * The record passed to TaskBroker for updating a checkpoint. - * Parameters to store the result of the executed checkpoint - * - * @public - */ -export type CheckpointRecord = - | { - key: string; - status: 'success'; - value: JsonValue; - } - | { - key: string; - status: 'failed'; - reason: string; - }; - /** * The state of all task's checkpoints * @@ -142,7 +124,14 @@ export interface TaskContext { cancelSignal: AbortSignal; spec: TaskSpec; secrets?: TaskSecrets; - state?: TaskState; + state?: { + [key: string]: + | { status: 'failed'; reason: string } + | { + status: 'success'; + value: JsonValue; + }; + }; createdBy?: string; done: boolean; isDryRun?: boolean; @@ -151,9 +140,33 @@ export interface TaskContext { emitLog(message: string, logMetadata?: JsonObject): Promise; - getCheckpoints?(): Promise<{ state: TaskState } | undefined>; + getTaskState?(): Promise< + | { + state: { + [key: string]: + | { status: 'failed'; reason: string } + | { + status: 'success'; + value: JsonValue; + }; + }; + } + | undefined + >; - updateCheckpoint?(options: CheckpointRecord): Promise; + updateCheckpoint?( + options: + | { + key: string; + status: 'success'; + value: JsonValue; + } + | { + key: string; + status: 'failed'; + reason: string; + }, + ): Promise; getWorkspaceName(): Promise; }