From 9990df8a1f5811e2c0f1163b685cfa1f2d46ba63 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 14 Oct 2021 17:38:51 +0100 Subject: [PATCH 01/16] Exports required to implement scaffolder broker Signed-off-by: Brian Fletcher --- .changeset/lovely-cars-sneeze.md | 5 + plugins/scaffolder-backend/api-report.md | 309 ++++++++++++++++++ .../src/scaffolder/index.ts | 24 ++ .../src/scaffolder/tasks/DatabaseTaskStore.ts | 8 + .../scaffolder/tasks/DefaultWorkflowRunner.ts | 4 + .../scaffolder/tasks/LegacyWorkflowRunner.ts | 4 + .../src/scaffolder/tasks/StorageTaskBroker.ts | 11 +- .../src/scaffolder/tasks/TaskWorker.ts | 4 + .../src/scaffolder/tasks/index.ts | 4 +- .../src/scaffolder/tasks/types.ts | 65 +++- .../scaffolder-backend/src/service/router.ts | 16 +- 11 files changed, 445 insertions(+), 9 deletions(-) create mode 100644 .changeset/lovely-cars-sneeze.md diff --git a/.changeset/lovely-cars-sneeze.md b/.changeset/lovely-cars-sneeze.md new file mode 100644 index 0000000000..8e6bfd691c --- /dev/null +++ b/.changeset/lovely-cars-sneeze.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Expose some classes and interfaces public so TaskWorkers can run externally from the scaffolder API. diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index ef4defcc50..e0e7ad7ee5 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -16,6 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import express from 'express'; import { JsonObject } from '@backstage/config'; import { JsonValue } from '@backstage/config'; +import { Knex } from 'knex'; import { LocationSpec } from '@backstage/catalog-model'; import { Logger as Logger_2 } from 'winston'; import { Octokit } from '@octokit/rest'; @@ -25,6 +26,7 @@ import { ScmIntegrationRegistry } from '@backstage/integration'; import { ScmIntegrations } from '@backstage/integration'; import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; import { UrlReader } from '@backstage/backend-common'; +import * as winston from 'winston'; import { Writable } from 'stream'; // Warning: (ae-forgotten-export) The symbol "InputBase" needs to be exported by the entry point index.d.ts @@ -189,6 +191,78 @@ export const createTemplateAction: < templateAction: TemplateAction, ) => TemplateAction; +// @public +export class DatabaseTaskStore implements TaskStore { + constructor(db: Knex); + // (undocumented) + claimTask(): Promise; + // (undocumented) + completeTask({ + taskId, + status, + eventBody, + }: { + taskId: string; + status: Status; + eventBody: JsonObject; + }): Promise; + // (undocumented) + static create(knex: Knex): Promise; + // (undocumented) + createTask( + spec: TaskSpec, + secrets?: TaskSecrets, + ): Promise<{ + taskId: string; + }>; + // (undocumented) + emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise; + // (undocumented) + getTask(taskId: string): Promise; + // (undocumented) + heartbeatTask(taskId: string): Promise; + // Warning: (ae-forgotten-export) The symbol "TaskStoreGetEventsOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + listEvents({ taskId, after }: TaskStoreGetEventsOptions): Promise<{ + events: DbTaskEventRow[]; + }>; + // (undocumented) + listStaleTasks({ timeoutS }: { timeoutS: number }): Promise<{ + tasks: { + taskId: string; + }[]; + }>; +} + +// @public +export type DbTaskRow = { + id: string; + spec: TaskSpec; + status: Status; + createdAt: string; + lastHeartbeatAt?: string; + secrets?: TaskSecrets; +}; + +// Warning: (ae-forgotten-export) The symbol "WorkflowRunner" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "DefaultWorkflowRunner" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class DefaultWorkflowRunner implements WorkflowRunner { + // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts + constructor(options: Options_3); + // Warning: (ae-forgotten-export) The symbol "WorkflowResponse" needs to be exported by the entry point index.d.ts + // + // (undocumented) + execute(task: Task): Promise; +} + +// @public +export type DispatchResult = { + taskId: string; +}; + // Warning: (ae-missing-release-tag) "fetchContents" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -206,6 +280,14 @@ export function fetchContents({ outputPath: string; }): Promise; +// @public +export class LegacyWorkflowRunner implements WorkflowRunner { + // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts + constructor(options: Options_2); + // (undocumented) + execute(task: Task): Promise; +} + // Warning: (ae-missing-release-tag) "OctokitProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -216,6 +298,15 @@ export class OctokitProvider { getOctokit(repoUrl: string): Promise; } +// @public +export type RawDbTaskEventRow = { + id: number; + task_id: string; + body: string; + event_type: TaskEventType; + created_at: string; +}; + // Warning: (ae-missing-release-tag) "RouterOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -235,6 +326,8 @@ export interface RouterOptions { // (undocumented) reader: UrlReader; // (undocumented) + taskBroker?: TaskBroker; + // (undocumented) taskWorkers?: number; } @@ -260,6 +353,218 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { validateEntityKind(entity: Entity): Promise; } +// @public +export type Status = + | 'open' + | 'processing' + | 'failed' + | 'cancelled' + | 'completed'; + +// @public +export class StorageTaskBroker implements TaskBroker { + constructor(storage: TaskStore, logger: Logger_2); + // (undocumented) + claim(): Promise; + // (undocumented) + dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; + // (undocumented) + get(taskId: string): Promise; + // (undocumented) + protected readonly logger: Logger_2; + // (undocumented) + observe( + options: { + taskId: string; + after: number | undefined; + }, + callback: ( + error: Error | undefined, + result: { + events: DbTaskEventRow[]; + }, + ) => void, + ): () => void; + // (undocumented) + protected readonly storage: TaskStore; + // (undocumented) + vacuumTasks(timeoutS: { timeoutS: number }): Promise; +} + +// @public +export interface Task { + // Warning: (ae-forgotten-export) The symbol "CompletedTaskState" needs to be exported by the entry point index.d.ts + // + // (undocumented) + complete(result: CompletedTaskState, metadata?: JsonValue): Promise; + // (undocumented) + done: boolean; + // (undocumented) + emitLog(message: string, metadata?: JsonValue): Promise; + // (undocumented) + getWorkspaceName(): Promise; + // (undocumented) + secrets?: TaskSecrets; + // (undocumented) + spec: TaskSpec; +} + +// Warning: (ae-missing-release-tag) "TaskAgent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class TaskAgent implements Task { + // (undocumented) + complete(result: CompletedTaskState, metadata?: JsonObject): Promise; + // Warning: (ae-forgotten-export) The symbol "TaskState" needs to be exported by the entry point index.d.ts + // + // (undocumented) + static create( + state: TaskState, + storage: TaskStore, + logger: Logger_2, + ): TaskAgent; + // (undocumented) + get done(): boolean; + // (undocumented) + emitLog(message: string, metadata?: JsonObject): Promise; + // (undocumented) + getWorkspaceName(): Promise; + // (undocumented) + get secrets(): TaskSecrets | undefined; + // (undocumented) + get spec(): TaskSpec; +} + +// @public +export interface TaskBroker { + // (undocumented) + claim(): Promise; + // (undocumented) + dispatch( + spec: TaskSpec, + secretTaskSecretss?: TaskSecrets, + ): Promise; + // (undocumented) + get(taskId: string): Promise; + // (undocumented) + observe( + options: { + taskId: string; + after: number | undefined; + }, + callback: ( + error: Error | undefined, + result: { + events: DbTaskEventRow[]; + }, + ) => void, + ): () => void; + // (undocumented) + vacuumTasks(timeoutS: { timeoutS: number }): Promise; +} + +// @public +export type TaskEventType = 'completion' | 'log'; + +// @public +export type TaskSecrets = { + token: string | undefined; +}; + +// @public +export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3; + +// @public +export interface TaskSpecV1beta2 { + // (undocumented) + apiVersion: 'backstage.io/v1beta2'; + // (undocumented) + baseUrl?: string; + // (undocumented) + output: { + [name: string]: string; + }; + // (undocumented) + steps: Array<{ + id: string; + name: string; + action: string; + input?: JsonObject; + if?: string | boolean; + }>; + // (undocumented) + values: JsonObject; +} + +// @public +export interface TaskSpecV1beta3 { + // (undocumented) + apiVersion: 'scaffolder.backstage.io/v1beta3'; + // (undocumented) + baseUrl?: string; + // (undocumented) + output: { + [name: string]: JsonValue; + }; + // (undocumented) + parameters: JsonObject; + // Warning: (ae-forgotten-export) The symbol "TaskStep" needs to be exported by the entry point index.d.ts + // + // (undocumented) + steps: TaskStep[]; +} + +// @public +export interface TaskStore { + // (undocumented) + claimTask(): Promise; + // (undocumented) + completeTask(options: { + taskId: string; + status: Status; + eventBody: JsonObject; + }): Promise; + // (undocumented) + createTask( + task: TaskSpec, + secrets?: TaskSecrets, + ): Promise<{ + taskId: string; + }>; + // (undocumented) + emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise; + // (undocumented) + getTask(taskId: string): Promise; + // (undocumented) + heartbeatTask(taskId: string): Promise; + // (undocumented) + listEvents({ taskId, after }: TaskStoreGetEventsOptions): Promise<{ + events: DbTaskEventRow[]; + }>; + // (undocumented) + listStaleTasks(options: { timeoutS: number }): Promise<{ + tasks: { + taskId: string; + }[]; + }>; +} + +// @public +export type TaskStoreEmitOptions = { + taskId: string; + body: JsonObject; +}; + +// @public +export class TaskWorker { + // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts + constructor(options: Options); + // (undocumented) + runOneTask(task: Task): Promise; + // (undocumented) + start(): void; +} + // Warning: (ae-missing-release-tag) "TemplateAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -286,4 +591,8 @@ export class TemplateActionRegistry { action: TemplateAction, ): void; } + +// Warnings were encountered during analysis: +// +// src/scaffolder/tasks/DatabaseTaskStore.d.ts:51:9 - (ae-forgotten-export) The symbol "DbTaskEventRow" needs to be exported by the entry point index.d.ts ``` diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index 284e372b4c..cf2a554189 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -14,3 +14,27 @@ * limitations under the License. */ export * from './actions'; +export { + TaskAgent, + DatabaseTaskStore, + StorageTaskBroker, + TaskWorker, + LegacyWorkflowRunner, + DefaultWorkflowRunner, +} from './tasks'; +export type { + Task, + TaskBroker, + TaskSpec, + TaskSecrets, + DispatchResult, + TaskStore, + TaskStoreEmitOptions, + DbTaskRow, + TaskSpecV1beta2, + TaskSpecV1beta3, + Status, + TaskEventType, +} from './tasks/types'; + +export type { RawDbTaskEventRow } from './tasks/DatabaseTaskStore'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 09cba596b1..8f659cad02 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -46,6 +46,10 @@ export type RawDbTaskRow = { secrets?: string; }; +/** + * RawDbTaskEventRow + * @public + */ export type RawDbTaskEventRow = { id: number; task_id: string; @@ -54,6 +58,10 @@ export type RawDbTaskEventRow = { created_at: string; }; +/** + * DatabaseTaskStore + * @public + */ export class DatabaseTaskStore implements TaskStore { static async create(knex: Knex): Promise { await knex.migrate.latest({ diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index 2ac309a1d4..7b2b437be2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -77,6 +77,10 @@ const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { return { taskLogger, streamLogger }; }; +/* + * DefaultWorkflowRunner + * @public + */ export class DefaultWorkflowRunner implements WorkflowRunner { private readonly nunjucks: nunjucks.Environment; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts index 97e3a4dec4..e71fe596cd 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts @@ -43,9 +43,13 @@ type Options = { const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta2 => taskSpec.apiVersion === 'backstage.io/v1beta2'; + /** * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced * with the default workflow runner interface in the future so this entire thing can go bye bye. + * + * LegacyWorkflowRunner + * @public */ export class LegacyWorkflowRunner implements WorkflowRunner { private readonly handlebars: typeof Handlebars; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 867e5bb127..59f605702d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -27,6 +27,9 @@ import { DbTaskRow, } from './types'; +/* + * @public + */ export class TaskAgent implements Task { private isDone = false; @@ -117,10 +120,14 @@ function defer() { return { promise, resolve }; } +/** + * StorageTaskBroker + * @public + */ export class StorageTaskBroker implements TaskBroker { constructor( - private readonly storage: TaskStore, - private readonly logger: Logger, + protected readonly storage: TaskStore, + protected readonly logger: Logger, ) {} private deferredDispatch = defer(); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 7122c2fc3e..1f6d9d3381 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -24,6 +24,10 @@ type Options = { }; }; +/** + * TaskWorker + * @public + */ export class TaskWorker { constructor(private readonly options: Options) {} start() { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index dd13264aed..c068e8d7e1 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -15,5 +15,7 @@ */ export { DatabaseTaskStore } from './DatabaseTaskStore'; -export { StorageTaskBroker } from './StorageTaskBroker'; +export { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; export { TaskWorker } from './TaskWorker'; +export { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; +export { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index cd93404165..63b409e7ca 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -16,6 +16,10 @@ import { JsonValue, JsonObject } from '@backstage/config'; +/** + * Status + * @public + */ export type Status = | 'open' | 'processing' @@ -25,6 +29,10 @@ export type Status = export type CompletedTaskState = 'failed' | 'completed'; +/** + * DbTaskRow + * @public + */ export type DbTaskRow = { id: string; spec: TaskSpec; @@ -34,7 +42,16 @@ export type DbTaskRow = { secrets?: TaskSecrets; }; +/** + * TaskEventType + * @public + */ export type TaskEventType = 'completion' | 'log'; + +/** + * DbTaskEventRow + * @public + */ export type DbTaskEventRow = { id: number; taskId: string; @@ -43,6 +60,10 @@ export type DbTaskEventRow = { createdAt: string; }; +/** + * TaskSpecV1beta2 + * @public + */ export interface TaskSpecV1beta2 { apiVersion: 'backstage.io/v1beta2'; baseUrl?: string; @@ -64,6 +85,11 @@ export interface TaskStep { input?: JsonObject; if?: string | boolean; } + +/** + * TaskSpecV1beta3 + * @public + */ export interface TaskSpecV1beta3 { apiVersion: 'scaffolder.backstage.io/v1beta3'; baseUrl?: string; @@ -72,16 +98,32 @@ export interface TaskSpecV1beta3 { output: { [name: string]: JsonValue }; } +/** + * TaskSpec + * @public + */ export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3; +/** + * TaskSecrets + * @public + */ export type TaskSecrets = { token: string | undefined; }; +/** + * DispatchResult + * @public + */ export type DispatchResult = { taskId: string; }; +/** + * Task + * @public + */ export interface Task { spec: TaskSpec; secrets?: TaskSecrets; @@ -91,9 +133,16 @@ export interface Task { getWorkspaceName(): Promise; } +/** + * TaskBroker + * @public + */ export interface TaskBroker { claim(): Promise; - dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; + dispatch( + spec: TaskSpec, + secretTaskSecretss?: TaskSecrets, + ): Promise; vacuumTasks(timeoutS: { timeoutS: number }): Promise; observe( options: { @@ -105,8 +154,14 @@ export interface TaskBroker { result: { events: DbTaskEventRow[] }, ) => void, ): () => void; + + get(taskId: string): Promise; } +/** + * TaskStoreEmitOptions + * @public + */ export type TaskStoreEmitOptions = { taskId: string; body: JsonObject; @@ -117,6 +172,10 @@ export type TaskStoreGetEventsOptions = { after?: number | undefined; }; +/** + * TaskStore + * @public + */ export interface TaskStore { createTask( task: TaskSpec, @@ -142,6 +201,10 @@ export interface TaskStore { } export type WorkflowResponse = { output: { [key: string]: JsonValue } }; +/* + * WorkflowRunner + * @public + */ export interface WorkflowRunner { execute(task: Task): Promise; } diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index f8ba99336f..5f855a6e88 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -25,7 +25,13 @@ import { StorageTaskBroker, TaskWorker, } from '../scaffolder/tasks'; -import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry'; +import { + TemplateActionRegistry, + TemplateAction, + createBuiltinActions, + TaskBroker, + TaskSpec, +} from '../scaffolder'; import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { ContainerRunner, @@ -38,11 +44,9 @@ import { TemplateEntityV1beta2, Entity } from '@backstage/catalog-model'; import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; import { ScmIntegrations } from '@backstage/integration'; -import { TemplateAction } from '../scaffolder/actions'; -import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; + import { LegacyWorkflowRunner } from '../scaffolder/tasks/LegacyWorkflowRunner'; import { DefaultWorkflowRunner } from '../scaffolder/tasks/DefaultWorkflowRunner'; -import { TaskSpec } from '../scaffolder/tasks/types'; export interface RouterOptions { logger: Logger; @@ -53,6 +57,7 @@ export interface RouterOptions { actions?: TemplateAction[]; taskWorkers?: number; containerRunner: ContainerRunner; + taskBroker?: TaskBroker; } function isSupportedTemplate( @@ -89,7 +94,8 @@ export async function createRouter( const databaseTaskStore = await DatabaseTaskStore.create( await database.getClient(), ); - const taskBroker = new StorageTaskBroker(databaseTaskStore, logger); + const taskBroker = + options.taskBroker || new StorageTaskBroker(databaseTaskStore, logger); const actionRegistry = new TemplateActionRegistry(); const legacyWorkflowRunner = new LegacyWorkflowRunner({ logger, From 01a4c5f9987937746b45dc2c98c35890d343e8ca Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Fri, 15 Oct 2021 10:36:49 +0100 Subject: [PATCH 02/16] fixes tests Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/src/service/router.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 5f855a6e88..afc9cee986 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -25,13 +25,8 @@ import { StorageTaskBroker, TaskWorker, } from '../scaffolder/tasks'; -import { - TemplateActionRegistry, - TemplateAction, - createBuiltinActions, - TaskBroker, - TaskSpec, -} from '../scaffolder'; +import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry'; +import { TaskBroker } from '../scaffolder'; import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { ContainerRunner, @@ -44,9 +39,12 @@ import { TemplateEntityV1beta2, Entity } from '@backstage/catalog-model'; import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; import { ScmIntegrations } from '@backstage/integration'; +import { TemplateAction } from '../scaffolder/actions'; +import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; import { LegacyWorkflowRunner } from '../scaffolder/tasks/LegacyWorkflowRunner'; import { DefaultWorkflowRunner } from '../scaffolder/tasks/DefaultWorkflowRunner'; +import { TaskSpec } from '../scaffolder/tasks/types'; export interface RouterOptions { logger: Logger; From 5a58092168be995a31d9d49bb26b7e70caec068c Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 21 Oct 2021 16:32:45 +0100 Subject: [PATCH 03/16] no longer expose the storage task broker We can implement the task broker completely, so it is not neccessary to export it. This commit also addresses some review comments. Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/api-report.md | 178 ++++++++---------- plugins/scaffolder-backend/src/index.ts | 7 + .../src/scaffolder/index.ts | 25 +-- .../src/scaffolder/tasks/DatabaseTaskStore.ts | 17 +- .../scaffolder/tasks/DefaultWorkflowRunner.ts | 8 +- .../scaffolder/tasks/LegacyWorkflowRunner.ts | 4 - .../tasks/StorageTaskBroker.test.ts | 6 +- .../src/scaffolder/tasks/StorageTaskBroker.ts | 27 +-- .../src/scaffolder/tasks/TaskWorker.ts | 58 +++++- .../src/scaffolder/tasks/index.ts | 4 +- .../src/scaffolder/tasks/types.ts | 56 +++--- .../scaffolder-backend/src/service/router.ts | 9 +- 12 files changed, 215 insertions(+), 184 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index e0e7ad7ee5..f47c550875 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -26,7 +26,6 @@ import { ScmIntegrationRegistry } from '@backstage/integration'; import { ScmIntegrations } from '@backstage/integration'; import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; import { UrlReader } from '@backstage/backend-common'; -import * as winston from 'winston'; import { Writable } from 'stream'; // Warning: (ae-forgotten-export) The symbol "InputBase" needs to be exported by the entry point index.d.ts @@ -57,6 +56,9 @@ export class CatalogEntityClient { ): Promise; } +// @public +export type CompletedTaskState = 'failed' | 'completed'; + // Warning: (ae-missing-release-tag) "createBuiltinActions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -191,11 +193,22 @@ export const createTemplateAction: < templateAction: TemplateAction, ) => TemplateAction; +// Warning: (ae-missing-release-tag) "CreateWorkerOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type CreateWorkerOptions = { + taskBroker: TaskBroker; + actionRegistry: TemplateActionRegistry; + integrations: ScmIntegrations; + workingDirectory: string; + logger: Logger_2; +}; + // @public export class DatabaseTaskStore implements TaskStore { constructor(db: Knex); // (undocumented) - claimTask(): Promise; + claimTask(): Promise; // (undocumented) completeTask({ taskId, @@ -218,14 +231,12 @@ export class DatabaseTaskStore implements TaskStore { // (undocumented) emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise; // (undocumented) - getTask(taskId: string): Promise; + getTask(taskId: string): Promise; // (undocumented) heartbeatTask(taskId: string): Promise; - // Warning: (ae-forgotten-export) The symbol "TaskStoreGetEventsOptions" needs to be exported by the entry point index.d.ts - // // (undocumented) - listEvents({ taskId, after }: TaskStoreGetEventsOptions): Promise<{ - events: DbTaskEventRow[]; + listEvents({ taskId, after }: TaskStoreListEventsOptions): Promise<{ + events: SerializedTaskEvent[]; }>; // (undocumented) listStaleTasks({ timeoutS }: { timeoutS: number }): Promise<{ @@ -235,29 +246,6 @@ export class DatabaseTaskStore implements TaskStore { }>; } -// @public -export type DbTaskRow = { - id: string; - spec: TaskSpec; - status: Status; - createdAt: string; - lastHeartbeatAt?: string; - secrets?: TaskSecrets; -}; - -// Warning: (ae-forgotten-export) The symbol "WorkflowRunner" needs to be exported by the entry point index.d.ts -// Warning: (ae-missing-release-tag) "DefaultWorkflowRunner" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export class DefaultWorkflowRunner implements WorkflowRunner { - // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts - constructor(options: Options_3); - // Warning: (ae-forgotten-export) The symbol "WorkflowResponse" needs to be exported by the entry point index.d.ts - // - // (undocumented) - execute(task: Task): Promise; -} - // @public export type DispatchResult = { taskId: string; @@ -280,14 +268,6 @@ export function fetchContents({ outputPath: string; }): Promise; -// @public -export class LegacyWorkflowRunner implements WorkflowRunner { - // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts - constructor(options: Options_2); - // (undocumented) - execute(task: Task): Promise; -} - // Warning: (ae-missing-release-tag) "OctokitProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -299,17 +279,6 @@ export class OctokitProvider { } // @public -export type RawDbTaskEventRow = { - id: number; - task_id: string; - body: string; - event_type: TaskEventType; - created_at: string; -}; - -// Warning: (ae-missing-release-tag) "RouterOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) export interface RouterOptions { // (undocumented) actions?: TemplateAction[]; @@ -353,6 +322,25 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { validateEntityKind(entity: Entity): Promise; } +// @public +export type SerializedTask = { + id: string; + spec: TaskSpec; + status: Status; + createdAt: string; + lastHeartbeatAt?: string; + secrets?: TaskSecrets; +}; + +// @public +export type SerializedTaskEvent = { + id: number; + taskId: string; + body: JsonObject; + type: TaskEventType; + createdAt: string; +}; + // @public export type Status = | 'open' @@ -361,40 +349,8 @@ export type Status = | 'cancelled' | 'completed'; -// @public -export class StorageTaskBroker implements TaskBroker { - constructor(storage: TaskStore, logger: Logger_2); - // (undocumented) - claim(): Promise; - // (undocumented) - dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; - // (undocumented) - get(taskId: string): Promise; - // (undocumented) - protected readonly logger: Logger_2; - // (undocumented) - observe( - options: { - taskId: string; - after: number | undefined; - }, - callback: ( - error: Error | undefined, - result: { - events: DbTaskEventRow[]; - }, - ) => void, - ): () => void; - // (undocumented) - protected readonly storage: TaskStore; - // (undocumented) - vacuumTasks(timeoutS: { timeoutS: number }): Promise; -} - // @public export interface Task { - // Warning: (ae-forgotten-export) The symbol "CompletedTaskState" needs to be exported by the entry point index.d.ts - // // (undocumented) complete(result: CompletedTaskState, metadata?: JsonValue): Promise; // (undocumented) @@ -409,14 +365,10 @@ export interface Task { spec: TaskSpec; } -// Warning: (ae-missing-release-tag) "TaskAgent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export class TaskAgent implements Task { // (undocumented) complete(result: CompletedTaskState, metadata?: JsonObject): Promise; - // Warning: (ae-forgotten-export) The symbol "TaskState" needs to be exported by the entry point index.d.ts - // // (undocumented) static create( state: TaskState, @@ -440,12 +392,9 @@ export interface TaskBroker { // (undocumented) claim(): Promise; // (undocumented) - dispatch( - spec: TaskSpec, - secretTaskSecretss?: TaskSecrets, - ): Promise; + dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; // (undocumented) - get(taskId: string): Promise; + get(taskId: string): Promise; // (undocumented) observe( options: { @@ -455,7 +404,7 @@ export interface TaskBroker { callback: ( error: Error | undefined, result: { - events: DbTaskEventRow[]; + events: SerializedTaskEvent[]; }, ) => void, ): () => void; @@ -463,9 +412,6 @@ export interface TaskBroker { vacuumTasks(timeoutS: { timeoutS: number }): Promise; } -// @public -export type TaskEventType = 'completion' | 'log'; - // @public export type TaskSecrets = { token: string | undefined; @@ -514,10 +460,20 @@ export interface TaskSpecV1beta3 { steps: TaskStep[]; } +// @public +export interface TaskState { + // (undocumented) + secrets?: TaskSecrets; + // (undocumented) + spec: TaskSpec; + // (undocumented) + taskId: string; +} + // @public export interface TaskStore { // (undocumented) - claimTask(): Promise; + claimTask(): Promise; // (undocumented) completeTask(options: { taskId: string; @@ -534,12 +490,12 @@ export interface TaskStore { // (undocumented) emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise; // (undocumented) - getTask(taskId: string): Promise; + getTask(taskId: string): Promise; // (undocumented) heartbeatTask(taskId: string): Promise; // (undocumented) - listEvents({ taskId, after }: TaskStoreGetEventsOptions): Promise<{ - events: DbTaskEventRow[]; + listEvents({ taskId, after }: TaskStoreListEventsOptions): Promise<{ + events: SerializedTaskEvent[]; }>; // (undocumented) listStaleTasks(options: { timeoutS: number }): Promise<{ @@ -555,16 +511,32 @@ export type TaskStoreEmitOptions = { body: JsonObject; }; +// @public +export type TaskStoreListEventsOptions = { + taskId: string; + after?: number | undefined; +}; + // @public export class TaskWorker { - // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts - constructor(options: Options); + constructor(options: TaskWorkerOptions); + // (undocumented) + static createWorker(options: CreateWorkerOptions): TaskWorker; // (undocumented) runOneTask(task: Task): Promise; // (undocumented) start(): void; } +// @public +export type TaskWorkerOptions = { + taskBroker: TaskBroker; + runners: { + legacyWorkflowRunner: LegacyWorkflowRunner; + workflowRunner: WorkflowRunner; + }; +}; + // Warning: (ae-missing-release-tag) "TemplateAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -594,5 +566,7 @@ export class TemplateActionRegistry { // Warnings were encountered during analysis: // -// src/scaffolder/tasks/DatabaseTaskStore.d.ts:51:9 - (ae-forgotten-export) The symbol "DbTaskEventRow" needs to be exported by the entry point index.d.ts +// src/scaffolder/tasks/TaskWorker.d.ts:14:9 - (ae-forgotten-export) The symbol "LegacyWorkflowRunner" needs to be exported by the entry point index.d.ts +// src/scaffolder/tasks/TaskWorker.d.ts:15:9 - (ae-forgotten-export) The symbol "WorkflowRunner" needs to be exported by the entry point index.d.ts +// src/scaffolder/tasks/types.d.ts:37:5 - (ae-forgotten-export) The symbol "TaskEventType" needs to be exported by the entry point index.d.ts ``` diff --git a/plugins/scaffolder-backend/src/index.ts b/plugins/scaffolder-backend/src/index.ts index 76cda63eef..4c31e9f982 100644 --- a/plugins/scaffolder-backend/src/index.ts +++ b/plugins/scaffolder-backend/src/index.ts @@ -24,3 +24,10 @@ export * from './scaffolder'; export * from './service/router'; export * from './lib/catalog'; export * from './processor'; +export { TaskAgent } from './scaffolder/tasks'; +export type { + TaskBroker, + DispatchResult, + TaskStore, + Task, +} from './scaffolder/tasks/types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index cf2a554189..b2599f20ed 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -14,27 +14,18 @@ * limitations under the License. */ export * from './actions'; -export { - TaskAgent, - DatabaseTaskStore, - StorageTaskBroker, - TaskWorker, - LegacyWorkflowRunner, - DefaultWorkflowRunner, -} from './tasks'; +export { DatabaseTaskStore, TaskWorker } from './tasks'; +export type { TaskWorkerOptions, CreateWorkerOptions } from './tasks'; +export type { TaskState } from './tasks'; export type { - Task, - TaskBroker, - TaskSpec, TaskSecrets, - DispatchResult, - TaskStore, + TaskSpec, + CompletedTaskState, TaskStoreEmitOptions, - DbTaskRow, + TaskStoreListEventsOptions, + SerializedTask, + SerializedTaskEvent, TaskSpecV1beta2, TaskSpecV1beta3, Status, - TaskEventType, } from './tasks/types'; - -export type { RawDbTaskEventRow } from './tasks/DatabaseTaskStore'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 8f659cad02..3aec10d949 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -20,15 +20,15 @@ import { ConflictError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; import { - DbTaskEventRow, - DbTaskRow, + SerializedTaskEvent, + SerializedTask, Status, TaskEventType, TaskSecrets, TaskSpec, TaskStore, TaskStoreEmitOptions, - TaskStoreGetEventsOptions, + TaskStoreListEventsOptions, } from './types'; import { DateTime } from 'luxon'; @@ -46,10 +46,6 @@ export type RawDbTaskRow = { secrets?: string; }; -/** - * RawDbTaskEventRow - * @public - */ export type RawDbTaskEventRow = { id: number; task_id: string; @@ -60,6 +56,7 @@ export type RawDbTaskEventRow = { /** * DatabaseTaskStore + * * @public */ export class DatabaseTaskStore implements TaskStore { @@ -72,7 +69,7 @@ export class DatabaseTaskStore implements TaskStore { constructor(private readonly db: Knex) {} - async getTask(taskId: string): Promise { + async getTask(taskId: string): Promise { const [result] = await this.db('tasks') .where({ id: taskId }) .select(); @@ -109,7 +106,7 @@ export class DatabaseTaskStore implements TaskStore { return { taskId }; } - async claimTask(): Promise { + async claimTask(): Promise { return this.db.transaction(async tx => { const [task] = await tx('tasks') .where({ @@ -251,7 +248,7 @@ export class DatabaseTaskStore implements TaskStore { async listEvents({ taskId, after, - }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }> { + }: TaskStoreListEventsOptions): Promise<{ events: SerializedTaskEvent[] }> { const rawEvents = await this.db('task_events') .where({ task_id: taskId, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index 7b2b437be2..145ded6bc0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -34,7 +34,7 @@ import { validate as validateJsonSchema } from 'jsonschema'; import { parseRepoUrl } from '../actions/builtin/publish/util'; import { TemplateActionRegistry } from '../actions'; -type Options = { +type NunjucksWorkflowRunnerOptions = { workingDirectory: string; actionRegistry: TemplateActionRegistry; integrations: ScmIntegrations; @@ -77,10 +77,6 @@ const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { return { taskLogger, streamLogger }; }; -/* - * DefaultWorkflowRunner - * @public - */ export class DefaultWorkflowRunner implements WorkflowRunner { private readonly nunjucks: nunjucks.Environment; @@ -92,7 +88,7 @@ export class DefaultWorkflowRunner implements WorkflowRunner { }, }; - constructor(private readonly options: Options) { + constructor(private readonly options: NunjucksWorkflowRunnerOptions) { this.nunjucks = nunjucks.configure(this.nunjucksOptions); // TODO(blam): let's work out how we can deprecate these. diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts index e71fe596cd..97e3a4dec4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts @@ -43,13 +43,9 @@ type Options = { const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta2 => taskSpec.apiVersion === 'backstage.io/v1beta2'; - /** * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced * with the default workflow runner interface in the future so this entire thing can go bye bye. - * - * LegacyWorkflowRunner - * @public */ export class LegacyWorkflowRunner implements WorkflowRunner { private readonly handlebars: typeof Handlebars; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts index 1a4d0d68c5..183253ea0e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -18,7 +18,7 @@ import { getVoidLogger, DatabaseManager } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { DatabaseTaskStore } from './DatabaseTaskStore'; import { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; -import { TaskSecrets, TaskSpec, DbTaskEventRow } from './types'; +import { TaskSecrets, TaskSpec, SerializedTaskEvent } from './types'; async function createStore(): Promise { const manager = DatabaseManager.fromConfig( @@ -127,8 +127,8 @@ describe('StorageTaskBroker', () => { const { taskId } = await broker1.dispatch({} as TaskSpec); - const logPromise = new Promise(resolve => { - const observedEvents = new Array(); + const logPromise = new Promise(resolve => { + const observedEvents = new Array(); broker2.observe({ taskId, after: undefined }, (_err, { events }) => { observedEvents.push(...events); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 59f605702d..af1f3752e1 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -23,11 +23,13 @@ import { TaskStore, TaskBroker, DispatchResult, - DbTaskEventRow, - DbTaskRow, + SerializedTaskEvent, + SerializedTask, } from './types'; -/* +/** + * TaskAgent + * * @public */ export class TaskAgent implements Task { @@ -106,7 +108,12 @@ export class TaskAgent implements Task { } } -interface TaskState { +/** + * TaskState + * + * @public + */ +export interface TaskState { spec: TaskSpec; taskId: string; secrets?: TaskSecrets; @@ -120,14 +127,10 @@ function defer() { return { promise, resolve }; } -/** - * StorageTaskBroker - * @public - */ export class StorageTaskBroker implements TaskBroker { constructor( - protected readonly storage: TaskStore, - protected readonly logger: Logger, + private readonly storage: TaskStore, + private readonly logger: Logger, ) {} private deferredDispatch = defer(); @@ -161,7 +164,7 @@ export class StorageTaskBroker implements TaskBroker { }; } - async get(taskId: string): Promise { + async get(taskId: string): Promise { return this.storage.getTask(taskId); } @@ -172,7 +175,7 @@ export class StorageTaskBroker implements TaskBroker { }, callback: ( error: Error | undefined, - result: { events: DbTaskEventRow[] }, + result: { events: SerializedTaskEvent[] }, ) => void, ): () => void { const { taskId } = options; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 1f6d9d3381..65ba71e9fe 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -13,10 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Task, TaskBroker, WorkflowRunner } from './types'; import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; +import { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; +import { Logger } from 'winston'; +import { TemplateActionRegistry } from '../actions'; +import { ScmIntegrations } from '@backstage/integration'; -type Options = { +/** + * TaskWorkerOptions + * + * @public + */ +export type TaskWorkerOptions = { taskBroker: TaskBroker; runners: { legacyWorkflowRunner: LegacyWorkflowRunner; @@ -24,12 +34,56 @@ type Options = { }; }; +/** + * CreateWorkerOptions + * + * @public + */ +export type CreateWorkerOptions = { + taskBroker: TaskBroker; + actionRegistry: TemplateActionRegistry; + integrations: ScmIntegrations; + workingDirectory: string; + logger: Logger; +}; + /** * TaskWorker + * * @public */ export class TaskWorker { - constructor(private readonly options: Options) {} + constructor(private readonly options: TaskWorkerOptions) {} + + static createWorker(options: CreateWorkerOptions) { + const { + taskBroker, + logger, + actionRegistry, + integrations, + workingDirectory, + } = options; + + const legacyWorkflowRunner = new LegacyWorkflowRunner({ + logger, + actionRegistry, + integrations, + workingDirectory, + }); + + const workflowRunner = new DefaultWorkflowRunner({ + actionRegistry, + integrations, + logger, + workingDirectory, + }); + + return new TaskWorker({ + taskBroker: taskBroker, + runners: { legacyWorkflowRunner, workflowRunner }, + }); + } + start() { (async () => { for (;;) { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index c068e8d7e1..ada12df476 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -16,6 +16,6 @@ export { DatabaseTaskStore } from './DatabaseTaskStore'; export { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; +export type { TaskState } from './StorageTaskBroker'; export { TaskWorker } from './TaskWorker'; -export { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; -export { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; +export type { TaskWorkerOptions, CreateWorkerOptions } from './TaskWorker'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 63b409e7ca..e30d156736 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -18,6 +18,7 @@ import { JsonValue, JsonObject } from '@backstage/config'; /** * Status + * * @public */ export type Status = @@ -27,13 +28,19 @@ export type Status = | 'cancelled' | 'completed'; +/** + * CompletedTaskState + * + * @public + */ export type CompletedTaskState = 'failed' | 'completed'; /** - * DbTaskRow + * SerializedTask + * * @public */ -export type DbTaskRow = { +export type SerializedTask = { id: string; spec: TaskSpec; status: Status; @@ -42,17 +49,14 @@ export type DbTaskRow = { secrets?: TaskSecrets; }; -/** - * TaskEventType - * @public - */ export type TaskEventType = 'completion' | 'log'; /** - * DbTaskEventRow + * SerializedTaskEvent + * * @public */ -export type DbTaskEventRow = { +export type SerializedTaskEvent = { id: number; taskId: string; body: JsonObject; @@ -62,6 +66,7 @@ export type DbTaskEventRow = { /** * TaskSpecV1beta2 + * * @public */ export interface TaskSpecV1beta2 { @@ -88,6 +93,7 @@ export interface TaskStep { /** * TaskSpecV1beta3 + * * @public */ export interface TaskSpecV1beta3 { @@ -100,12 +106,14 @@ export interface TaskSpecV1beta3 { /** * TaskSpec + * * @public */ export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3; /** * TaskSecrets + * * @public */ export type TaskSecrets = { @@ -114,6 +122,7 @@ export type TaskSecrets = { /** * DispatchResult + * * @public */ export type DispatchResult = { @@ -122,6 +131,7 @@ export type DispatchResult = { /** * Task + * * @public */ export interface Task { @@ -135,14 +145,12 @@ export interface Task { /** * TaskBroker + * * @public */ export interface TaskBroker { claim(): Promise; - dispatch( - spec: TaskSpec, - secretTaskSecretss?: TaskSecrets, - ): Promise; + dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; vacuumTasks(timeoutS: { timeoutS: number }): Promise; observe( options: { @@ -151,15 +159,15 @@ export interface TaskBroker { }, callback: ( error: Error | undefined, - result: { events: DbTaskEventRow[] }, + result: { events: SerializedTaskEvent[] }, ) => void, ): () => void; - - get(taskId: string): Promise; + get(taskId: string): Promise; } /** * TaskStoreEmitOptions + * * @public */ export type TaskStoreEmitOptions = { @@ -167,13 +175,19 @@ export type TaskStoreEmitOptions = { body: JsonObject; }; -export type TaskStoreGetEventsOptions = { +/** + * TaskStoreListEventsOptions + * + * @public + */ +export type TaskStoreListEventsOptions = { taskId: string; after?: number | undefined; }; /** * TaskStore + * * @public */ export interface TaskStore { @@ -181,8 +195,8 @@ export interface TaskStore { task: TaskSpec, secrets?: TaskSecrets, ): Promise<{ taskId: string }>; - getTask(taskId: string): Promise; - claimTask(): Promise; + getTask(taskId: string): Promise; + claimTask(): Promise; completeTask(options: { taskId: string; status: Status; @@ -197,14 +211,10 @@ export interface TaskStore { listEvents({ taskId, after, - }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }>; + }: TaskStoreListEventsOptions): Promise<{ events: SerializedTaskEvent[] }>; } export type WorkflowResponse = { output: { [key: string]: JsonValue } }; -/* - * WorkflowRunner - * @public - */ export interface WorkflowRunner { execute(task: Task): Promise; } diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index afc9cee986..d4d4dd0486 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -26,7 +26,6 @@ import { TaskWorker, } from '../scaffolder/tasks'; import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry'; -import { TaskBroker } from '../scaffolder'; import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { ContainerRunner, @@ -41,11 +40,15 @@ import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; import { ScmIntegrations } from '@backstage/integration'; import { TemplateAction } from '../scaffolder/actions'; import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; - import { LegacyWorkflowRunner } from '../scaffolder/tasks/LegacyWorkflowRunner'; import { DefaultWorkflowRunner } from '../scaffolder/tasks/DefaultWorkflowRunner'; -import { TaskSpec } from '../scaffolder/tasks/types'; +import { TaskBroker, TaskSpec } from '../scaffolder/tasks/types'; +/** + * RouterOptions + * + * @public + */ export interface RouterOptions { logger: Logger; config: Config; From 33d99f7cdd963da3421b4570446641dd0dc890dd Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Fri, 22 Oct 2021 08:12:29 +0100 Subject: [PATCH 04/16] data store options type added Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/api-report.md | 10 ++++---- .../src/scaffolder/tasks/DatabaseTaskStore.ts | 23 +++++++++++++++---- .../tasks/StorageTaskBroker.test.ts | 4 +++- .../src/scaffolder/tasks/TaskWorker.test.ts | 4 +++- .../src/scaffolder/tasks/index.ts | 2 +- .../scaffolder-backend/src/service/router.ts | 6 ++--- 6 files changed, 34 insertions(+), 15 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index f47c550875..deb2657c59 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -193,9 +193,7 @@ export const createTemplateAction: < templateAction: TemplateAction, ) => TemplateAction; -// Warning: (ae-missing-release-tag) "CreateWorkerOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export type CreateWorkerOptions = { taskBroker: TaskBroker; actionRegistry: TemplateActionRegistry; @@ -206,7 +204,7 @@ export type CreateWorkerOptions = { // @public export class DatabaseTaskStore implements TaskStore { - constructor(db: Knex); + constructor(options: DatabaseTaskStoreOptions); // (undocumented) claimTask(): Promise; // (undocumented) @@ -219,8 +217,10 @@ export class DatabaseTaskStore implements TaskStore { status: Status; eventBody: JsonObject; }): Promise; + // Warning: (ae-forgotten-export) The symbol "DatabaseTaskStoreOptions" needs to be exported by the entry point index.d.ts + // // (undocumented) - static create(knex: Knex): Promise; + static create(options: DatabaseTaskStoreOptions): Promise; // (undocumented) createTask( spec: TaskSpec, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 3aec10d949..6783da674e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -54,20 +54,35 @@ export type RawDbTaskEventRow = { created_at: string; }; +/** + * DatabaseTaskStore + * + * @public + */ +export type DatabaseTaskStoreOptions = { + database: Knex; +}; + /** * DatabaseTaskStore * * @public */ export class DatabaseTaskStore implements TaskStore { - static async create(knex: Knex): Promise { - await knex.migrate.latest({ + private readonly db: Knex; + + static async create( + options: DatabaseTaskStoreOptions, + ): Promise { + await options.database.migrate.latest({ directory: migrationsDir, }); - return new DatabaseTaskStore(knex); + return new DatabaseTaskStore(options); } - constructor(private readonly db: Knex) {} + constructor(options: DatabaseTaskStoreOptions) { + this.db = options.database; + } async getTask(taskId: string): Promise { const [result] = await this.db('tasks') diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts index 183253ea0e..c5d94c1324 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -31,7 +31,9 @@ async function createStore(): Promise { }, }), ).forPlugin('scaffolder'); - return await DatabaseTaskStore.create(await manager.getClient()); + return await DatabaseTaskStore.create({ + database: await manager.getClient(), + }); } describe('StorageTaskBroker', () => { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index 87a0229b5d..f6b332b6d3 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -33,7 +33,9 @@ async function createStore(): Promise { }, }), ).forPlugin('scaffolder'); - return await DatabaseTaskStore.create(await manager.getClient()); + return await DatabaseTaskStore.create({ + database: await manager.getClient(), + }); } describe('TaskWorker', () => { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index ada12df476..13831ca1bb 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - export { DatabaseTaskStore } from './DatabaseTaskStore'; +export type { DatabaseTaskStoreOptions } from './DatabaseTaskStore'; export { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; export type { TaskState } from './StorageTaskBroker'; export { TaskWorker } from './TaskWorker'; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index d4d4dd0486..0bc00a2778 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -92,9 +92,9 @@ export async function createRouter( const entityClient = new CatalogEntityClient(catalogClient); const integrations = ScmIntegrations.fromConfig(config); - const databaseTaskStore = await DatabaseTaskStore.create( - await database.getClient(), - ); + const databaseTaskStore = await DatabaseTaskStore.create({ + database: await database.getClient(), + }); const taskBroker = options.taskBroker || new StorageTaskBroker(databaseTaskStore, logger); const actionRegistry = new TemplateActionRegistry(); From 45a2a91a3165aca21aed793f7744326c9a55e29b Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Tue, 26 Oct 2021 09:48:56 +0100 Subject: [PATCH 05/16] update api reports after merge Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/api-report.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index b40270f317..a94fa8a2a0 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -14,11 +14,9 @@ import { createFetchCookiecutterAction } from '@backstage/plugin-scaffolder-back import { createPullRequest } from 'octokit-plugin-create-pull-request'; import { Entity } from '@backstage/catalog-model'; import express from 'express'; -import { JsonObject } from '@backstage/config'; -import { JsonObject as JsonObject_2 } from '@backstage/types'; -import { JsonValue } from '@backstage/config'; +import { JsonObject } from '@backstage/types'; +import { JsonValue } from '@backstage/types'; import { Knex } from 'knex'; -import { JsonValue as JsonValue_2 } from '@backstage/types'; import { LocationSpec } from '@backstage/catalog-model'; import { Logger as Logger_2 } from 'winston'; import { Octokit } from '@octokit/rest'; From d781df18437df3eb15149ce292158b0527f637ac Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Tue, 26 Oct 2021 11:34:13 +0100 Subject: [PATCH 06/16] hide workflow runners behind createWorker Also renames legacy and default workflow runners. Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/api-report.md | 19 +---- .../sample-templates/all-templates.yaml | 1 - .../bitbucket-demo/template/catalog-info.yaml | 4 +- .../src/scaffolder/index.ts | 3 +- .../tasks/DefaultWorkflowRunner.test.ts | 6 +- ...wRunner.ts => HandlebarsWorkflowRunner.ts} | 2 +- .../tasks/LegacyWorkflowRunner.test.ts | 6 +- ...lowRunner.ts => NunjucksWorkflowRunner.ts} | 2 +- .../src/scaffolder/tasks/TaskWorker.test.ts | 74 +++++++++++++------ .../src/scaffolder/tasks/TaskWorker.ts | 12 +-- .../src/scaffolder/tasks/types.ts | 5 ++ .../scaffolder-backend/src/service/router.ts | 25 ++----- 12 files changed, 82 insertions(+), 77 deletions(-) rename plugins/scaffolder-backend/src/scaffolder/tasks/{LegacyWorkflowRunner.ts => HandlebarsWorkflowRunner.ts} (99%) rename plugins/scaffolder-backend/src/scaffolder/tasks/{DefaultWorkflowRunner.ts => NunjucksWorkflowRunner.ts} (99%) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index a94fa8a2a0..d0f4cb36e8 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -412,6 +412,9 @@ export interface TaskBroker { vacuumTasks(timeoutS: { timeoutS: number }): Promise; } +// @public +export type TaskEventType = 'completion' | 'log'; + // @public export type TaskSecrets = { token: string | undefined; @@ -519,7 +522,6 @@ export type TaskStoreListEventsOptions = { // @public export class TaskWorker { - constructor(options: TaskWorkerOptions); // (undocumented) static createWorker(options: CreateWorkerOptions): TaskWorker; // (undocumented) @@ -528,15 +530,6 @@ export class TaskWorker { start(): void; } -// @public -export type TaskWorkerOptions = { - taskBroker: TaskBroker; - runners: { - legacyWorkflowRunner: LegacyWorkflowRunner; - workflowRunner: WorkflowRunner; - }; -}; - // Warning: (ae-missing-release-tag) "TemplateAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -563,10 +556,4 @@ export class TemplateActionRegistry { action: TemplateAction, ): void; } - -// Warnings were encountered during analysis: -// -// src/scaffolder/tasks/TaskWorker.d.ts:14:9 - (ae-forgotten-export) The symbol "LegacyWorkflowRunner" needs to be exported by the entry point index.d.ts -// src/scaffolder/tasks/TaskWorker.d.ts:15:9 - (ae-forgotten-export) The symbol "WorkflowRunner" needs to be exported by the entry point index.d.ts -// src/scaffolder/tasks/types.d.ts:37:5 - (ae-forgotten-export) The symbol "TaskEventType" needs to be exported by the entry point index.d.ts ``` diff --git a/plugins/scaffolder-backend/sample-templates/all-templates.yaml b/plugins/scaffolder-backend/sample-templates/all-templates.yaml index 2d5adb8e3b..6637c9479b 100644 --- a/plugins/scaffolder-backend/sample-templates/all-templates.yaml +++ b/plugins/scaffolder-backend/sample-templates/all-templates.yaml @@ -6,7 +6,6 @@ metadata: spec: targets: - ./remote-templates.yaml - # For local development of a template, you can reference your local templates here. # Examples: # diff --git a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml index 875664d2a8..3b685a57e0 100644 --- a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml +++ b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml @@ -1,8 +1,8 @@ apiVersion: backstage.io/v1alpha1 kind: Component metadata: - name: {{cookiecutter.name | jsonify}} + name: { { cookiecutter.name | jsonify } } spec: type: website lifecycle: experimental - owner: {{cookiecutter.owner | jsonify}} + owner: { { cookiecutter.owner | jsonify } } diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index b2599f20ed..6b0e033b7d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -15,7 +15,7 @@ */ export * from './actions'; export { DatabaseTaskStore, TaskWorker } from './tasks'; -export type { TaskWorkerOptions, CreateWorkerOptions } from './tasks'; +export type { CreateWorkerOptions } from './tasks'; export type { TaskState } from './tasks'; export type { TaskSecrets, @@ -28,4 +28,5 @@ export type { TaskSpecV1beta2, TaskSpecV1beta3, Status, + TaskEventType, } from './tasks/types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts index d6d1b0cfd2..0a843f4ddf 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -18,7 +18,7 @@ import mockFs from 'mock-fs'; import * as winston from 'winston'; import { getVoidLogger } from '@backstage/backend-common'; -import { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; +import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner'; import { TemplateActionRegistry } from '../actions'; import { ScmIntegrations } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; @@ -27,7 +27,7 @@ import { Task, TaskSpec } from './types'; describe('DefaultWorkflowRunner', () => { const logger = getVoidLogger(); let actionRegistry = new TemplateActionRegistry(); - let runner: DefaultWorkflowRunner; + let runner: NunjucksWorkflowRunner; let fakeActionHandler: jest.Mock; const integrations = ScmIntegrations.fromConfig( @@ -88,7 +88,7 @@ describe('DefaultWorkflowRunner', () => { }, }); - runner = new DefaultWorkflowRunner({ + runner = new NunjucksWorkflowRunner({ actionRegistry, integrations, workingDirectory: '/tmp', diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts similarity index 99% rename from plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts rename to plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts index 6d43ae978c..831ef98784 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts @@ -47,7 +47,7 @@ const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta2 => * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced * with the default workflow runner interface in the future so this entire thing can go bye bye. */ -export class LegacyWorkflowRunner implements WorkflowRunner { +export class HandlebarsWorkflowRunner implements WorkflowRunner { private readonly handlebars: typeof Handlebars; constructor(private readonly options: Options) { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts index 2714f1b80c..aa46747c68 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts @@ -20,12 +20,12 @@ import { createTemplateAction, TemplateActionRegistry } from '../actions'; import { ScmIntegrations } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; -import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; +import { HandlebarsWorkflowRunner } from './HandlebarsWorkflowRunner'; import { Task, TaskSpec } from './types'; import { RepoSpec } from '../actions/builtin/publish/util'; describe('LegacyWorkflowRunner', () => { - let runner: LegacyWorkflowRunner; + let runner: HandlebarsWorkflowRunner; const logger = getVoidLogger(); let actionRegistry = new TemplateActionRegistry(); @@ -60,7 +60,7 @@ describe('LegacyWorkflowRunner', () => { }, }); - runner = new LegacyWorkflowRunner({ + runner = new HandlebarsWorkflowRunner({ actionRegistry, integrations, workingDirectory: '/tmp', diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts similarity index 99% rename from plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts rename to plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index e56befd939..5cceb16324 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -77,7 +77,7 @@ const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { return { taskLogger, streamLogger }; }; -export class DefaultWorkflowRunner implements WorkflowRunner { +export class NunjucksWorkflowRunner implements WorkflowRunner { private readonly nunjucks: nunjucks.Environment; private readonly nunjucksOptions: nunjucks.ConfigureOptions = { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index f6b332b6d3..3cb73abb85 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -19,8 +19,20 @@ import { ConfigReader } from '@backstage/config'; import { DatabaseTaskStore } from './DatabaseTaskStore'; import { StorageTaskBroker } from './StorageTaskBroker'; import { TaskWorker } from './TaskWorker'; -import { WorkflowRunner } from './types'; -import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; +import { HandlebarsWorkflowRunner } from './HandlebarsWorkflowRunner'; +import { ScmIntegrations } from '@backstage/integration'; +import { TemplateActionRegistry } from '../actions'; +import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner'; + +jest.mock('./HandlebarsWorkflowRunner'); +const MockedHandlebarsWorkflowRunner = + HandlebarsWorkflowRunner as jest.Mock; +MockedHandlebarsWorkflowRunner.mockImplementation(); + +jest.mock('./NunjucksWorkflowRunner'); +const MockedNunjucksWorkflowRunner = + NunjucksWorkflowRunner as jest.Mock; +MockedNunjucksWorkflowRunner.mockImplementation(); async function createStore(): Promise { const manager = DatabaseManager.fromConfig( @@ -40,13 +52,19 @@ async function createStore(): Promise { describe('TaskWorker', () => { let storage: DatabaseTaskStore; - const workflowRunner: WorkflowRunner = { - execute: jest.fn(), - } as unknown as WorkflowRunner; - const legacyWorkflowRunner: LegacyWorkflowRunner = { + const integrations: ScmIntegrations = {} as ScmIntegrations; + + const actionRegistry: TemplateActionRegistry = {} as TemplateActionRegistry; + const workingDirectory = '/tmp/scaffolder'; + + const handlebarsWorkflowRunner: HandlebarsWorkflowRunner = { execute: jest.fn(), - } as unknown as LegacyWorkflowRunner; + } as unknown as HandlebarsWorkflowRunner; + + const workflowRunner: NunjucksWorkflowRunner = { + execute: jest.fn(), + } as unknown as NunjucksWorkflowRunner; beforeAll(async () => { storage = await createStore(); @@ -54,18 +72,22 @@ describe('TaskWorker', () => { beforeEach(() => { jest.resetAllMocks(); + MockedHandlebarsWorkflowRunner.mockImplementation( + () => handlebarsWorkflowRunner, + ); + MockedNunjucksWorkflowRunner.mockImplementation(() => workflowRunner); }); const logger = getVoidLogger(); it('should call the legacy workflow runner when the apiVersion is not beta3', async () => { const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ + const taskWorker = TaskWorker.createWorker({ + logger, + workingDirectory, + integrations, taskBroker: broker, - runners: { - legacyWorkflowRunner, - workflowRunner, - }, + actionRegistry, }); await broker.dispatch({ @@ -80,17 +102,23 @@ describe('TaskWorker', () => { const task = await broker.claim(); await taskWorker.runOneTask(task); - expect(legacyWorkflowRunner.execute).toHaveBeenCalled(); + expect(MockedHandlebarsWorkflowRunner).toBeCalledWith({ + actionRegistry, + integrations, + logger, + workingDirectory, + }); + expect(handlebarsWorkflowRunner.execute).toHaveBeenCalled(); }); it('should call the default workflow runner when the apiVersion is beta3', async () => { const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ + const taskWorker = TaskWorker.createWorker({ + logger, + workingDirectory, + integrations, taskBroker: broker, - runners: { - legacyWorkflowRunner, - workflowRunner, - }, + actionRegistry, }); await broker.dispatch({ @@ -114,12 +142,12 @@ describe('TaskWorker', () => { }); const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ + const taskWorker = TaskWorker.createWorker({ + logger, + workingDirectory, + integrations, taskBroker: broker, - runners: { - legacyWorkflowRunner, - workflowRunner, - }, + actionRegistry, }); const { taskId } = await broker.dispatch({ diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 6e70ea06a6..fcc215be99 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -15,8 +15,8 @@ */ import { Task, TaskBroker, WorkflowRunner } from './types'; -import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; -import { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; +import { HandlebarsWorkflowRunner } from './HandlebarsWorkflowRunner'; +import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner'; import { Logger } from 'winston'; import { TemplateActionRegistry } from '../actions'; import { ScmIntegrations } from '@backstage/integration'; @@ -30,7 +30,7 @@ import { assertError } from '@backstage/errors'; export type TaskWorkerOptions = { taskBroker: TaskBroker; runners: { - legacyWorkflowRunner: LegacyWorkflowRunner; + legacyWorkflowRunner: HandlebarsWorkflowRunner; workflowRunner: WorkflowRunner; }; }; @@ -54,7 +54,7 @@ export type CreateWorkerOptions = { * @public */ export class TaskWorker { - constructor(private readonly options: TaskWorkerOptions) {} + private constructor(private readonly options: TaskWorkerOptions) {} static createWorker(options: CreateWorkerOptions) { const { @@ -65,14 +65,14 @@ export class TaskWorker { workingDirectory, } = options; - const legacyWorkflowRunner = new LegacyWorkflowRunner({ + const legacyWorkflowRunner = new HandlebarsWorkflowRunner({ logger, actionRegistry, integrations, workingDirectory, }); - const workflowRunner = new DefaultWorkflowRunner({ + const workflowRunner = new NunjucksWorkflowRunner({ actionRegistry, integrations, logger, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index cd4d291719..f2e13923ec 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -49,6 +49,11 @@ export type SerializedTask = { secrets?: TaskSecrets; }; +/** + * TaskEventType + * + * @public + */ export type TaskEventType = 'completion' | 'log'; /** diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 0bc00a2778..fcd5551be2 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -40,8 +40,6 @@ import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; import { ScmIntegrations } from '@backstage/integration'; import { TemplateAction } from '../scaffolder/actions'; import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; -import { LegacyWorkflowRunner } from '../scaffolder/tasks/LegacyWorkflowRunner'; -import { DefaultWorkflowRunner } from '../scaffolder/tasks/DefaultWorkflowRunner'; import { TaskBroker, TaskSpec } from '../scaffolder/tasks/types'; /** @@ -98,28 +96,15 @@ export async function createRouter( const taskBroker = options.taskBroker || new StorageTaskBroker(databaseTaskStore, logger); const actionRegistry = new TemplateActionRegistry(); - const legacyWorkflowRunner = new LegacyWorkflowRunner({ - logger, - actionRegistry, - integrations, - workingDirectory, - }); - - const workflowRunner = new DefaultWorkflowRunner({ - actionRegistry, - integrations, - logger, - workingDirectory, - }); const workers = []; for (let i = 0; i < (taskWorkers || 1); i++) { - const worker = new TaskWorker({ + const worker = TaskWorker.createWorker({ taskBroker, - runners: { - legacyWorkflowRunner, - workflowRunner, - }, + actionRegistry, + integrations, + logger, + workingDirectory, }); workers.push(worker); } From c754c390ee16c4a89a6609fc057772ae9ffd9217 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Tue, 26 Oct 2021 12:09:54 +0100 Subject: [PATCH 07/16] fix up exports to follow covention Signed-off-by: Brian Fletcher --- .../scaffolder-backend/src/scaffolder/index.ts | 17 +---------------- .../src/scaffolder/tasks/index.ts | 18 +++++++++++++++--- .../scaffolder-backend/src/service/router.ts | 10 +++++----- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index 6b0e033b7d..e055223d8c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -14,19 +14,4 @@ * limitations under the License. */ export * from './actions'; -export { DatabaseTaskStore, TaskWorker } from './tasks'; -export type { CreateWorkerOptions } from './tasks'; -export type { TaskState } from './tasks'; -export type { - TaskSecrets, - TaskSpec, - CompletedTaskState, - TaskStoreEmitOptions, - TaskStoreListEventsOptions, - SerializedTask, - SerializedTaskEvent, - TaskSpecV1beta2, - TaskSpecV1beta3, - Status, - TaskEventType, -} from './tasks/types'; +export * from './tasks'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index 13831ca1bb..d99cbcda6e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -14,8 +14,20 @@ * limitations under the License. */ export { DatabaseTaskStore } from './DatabaseTaskStore'; -export type { DatabaseTaskStoreOptions } from './DatabaseTaskStore'; -export { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; +export { TaskAgent } from './StorageTaskBroker'; export type { TaskState } from './StorageTaskBroker'; export { TaskWorker } from './TaskWorker'; -export type { TaskWorkerOptions, CreateWorkerOptions } from './TaskWorker'; +export type { CreateWorkerOptions } from './TaskWorker'; +export type { + TaskSecrets, + TaskSpec, + CompletedTaskState, + TaskStoreEmitOptions, + TaskStoreListEventsOptions, + SerializedTask, + SerializedTaskEvent, + TaskSpecV1beta2, + TaskSpecV1beta3, + Status, + TaskEventType, +} from './types'; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index fcd5551be2..700f2c6e6f 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -22,10 +22,12 @@ import { CatalogEntityClient } from '../lib/catalog'; import { validate } from 'jsonschema'; import { DatabaseTaskStore, - StorageTaskBroker, + TemplateActionRegistry, TaskWorker, -} from '../scaffolder/tasks'; -import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry'; + TemplateAction, + createBuiltinActions, +} from '../scaffolder'; +import { StorageTaskBroker } from '../scaffolder/tasks/StorageTaskBroker'; import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { ContainerRunner, @@ -38,8 +40,6 @@ import { TemplateEntityV1beta2, Entity } from '@backstage/catalog-model'; import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; import { ScmIntegrations } from '@backstage/integration'; -import { TemplateAction } from '../scaffolder/actions'; -import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; import { TaskBroker, TaskSpec } from '../scaffolder/tasks/types'; /** From f49099fc13ea584f8d6807c4b6734ebc7d08db75 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Tue, 26 Oct 2021 15:42:02 +0100 Subject: [PATCH 08/16] fix more imports to follow convention Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/src/index.ts | 7 ------- plugins/scaffolder-backend/src/scaffolder/tasks/index.ts | 4 ++++ plugins/scaffolder-backend/src/service/router.test.ts | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/plugins/scaffolder-backend/src/index.ts b/plugins/scaffolder-backend/src/index.ts index 4c31e9f982..76cda63eef 100644 --- a/plugins/scaffolder-backend/src/index.ts +++ b/plugins/scaffolder-backend/src/index.ts @@ -24,10 +24,3 @@ export * from './scaffolder'; export * from './service/router'; export * from './lib/catalog'; export * from './processor'; -export { TaskAgent } from './scaffolder/tasks'; -export type { - TaskBroker, - DispatchResult, - TaskStore, - Task, -} from './scaffolder/tasks/types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index d99cbcda6e..941c27ef0e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -30,4 +30,8 @@ export type { TaskSpecV1beta3, Status, TaskEventType, + TaskBroker, + Task, + TaskStore, + DispatchResult, } from './types'; diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index c0fdcb2ef7..71b6d27b5e 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -40,7 +40,7 @@ import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; -import { createRouter } from './router'; +import { createRouter } from '../index'; const createCatalogClient = (templates: any[] = []) => ({ From 50eba2374b4d4ce2033714a94b0fe6b056adde9f Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 27 Oct 2021 09:16:23 +0100 Subject: [PATCH 09/16] revert prettier change to scaffolder examples Signed-off-by: Brian Fletcher --- .../bitbucket-demo/template/catalog-info.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml index 3b685a57e0..875664d2a8 100644 --- a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml +++ b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml @@ -1,8 +1,8 @@ apiVersion: backstage.io/v1alpha1 kind: Component metadata: - name: { { cookiecutter.name | jsonify } } + name: {{cookiecutter.name | jsonify}} spec: type: website lifecycle: experimental - owner: { { cookiecutter.owner | jsonify } } + owner: {{cookiecutter.owner | jsonify}} From 5a98c405f52b070918afb44f7116ecbaa6a6432a Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 27 Oct 2021 10:48:17 +0100 Subject: [PATCH 10/16] rename task agent to task manager Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/api-report.md | 80 +++++++++---------- .../tasks/DefaultWorkflowRunner.test.ts | 4 +- .../tasks/HandlebarsWorkflowRunner.ts | 4 +- .../tasks/LegacyWorkflowRunner.test.ts | 4 +- .../tasks/NunjucksWorkflowRunner.ts | 12 ++- .../tasks/StorageTaskBroker.test.ts | 12 +-- .../src/scaffolder/tasks/StorageTaskBroker.ts | 12 +-- .../src/scaffolder/tasks/TaskWorker.ts | 4 +- .../src/scaffolder/tasks/index.ts | 4 +- .../src/scaffolder/tasks/types.ts | 6 +- 10 files changed, 74 insertions(+), 68 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index d0f4cb36e8..5441c4de6b 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -349,48 +349,10 @@ export type Status = | 'cancelled' | 'completed'; -// @public -export interface Task { - // (undocumented) - complete(result: CompletedTaskState, metadata?: JsonValue): Promise; - // (undocumented) - done: boolean; - // (undocumented) - emitLog(message: string, metadata?: JsonValue): Promise; - // (undocumented) - getWorkspaceName(): Promise; - // (undocumented) - secrets?: TaskSecrets; - // (undocumented) - spec: TaskSpec; -} - -// @public -export class TaskAgent implements Task { - // (undocumented) - complete(result: CompletedTaskState, metadata?: JsonObject): Promise; - // (undocumented) - static create( - state: TaskState, - storage: TaskStore, - logger: Logger_2, - ): TaskAgent; - // (undocumented) - get done(): boolean; - // (undocumented) - emitLog(message: string, metadata?: JsonObject): Promise; - // (undocumented) - getWorkspaceName(): Promise; - // (undocumented) - get secrets(): TaskSecrets | undefined; - // (undocumented) - get spec(): TaskSpec; -} - // @public export interface TaskBroker { // (undocumented) - claim(): Promise; + claim(): Promise; // (undocumented) dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; // (undocumented) @@ -412,9 +374,47 @@ export interface TaskBroker { vacuumTasks(timeoutS: { timeoutS: number }): Promise; } +// @public +export interface TaskContext { + // (undocumented) + complete(result: CompletedTaskState, metadata?: JsonValue): Promise; + // (undocumented) + done: boolean; + // (undocumented) + emitLog(message: string, metadata?: JsonValue): Promise; + // (undocumented) + getWorkspaceName(): Promise; + // (undocumented) + secrets?: TaskSecrets; + // (undocumented) + spec: TaskSpec; +} + // @public export type TaskEventType = 'completion' | 'log'; +// @public +export class TaskManager implements TaskContext { + // (undocumented) + complete(result: CompletedTaskState, metadata?: JsonObject): Promise; + // (undocumented) + static create( + state: TaskState, + storage: TaskStore, + logger: Logger_2, + ): TaskManager; + // (undocumented) + get done(): boolean; + // (undocumented) + emitLog(message: string, metadata?: JsonObject): Promise; + // (undocumented) + getWorkspaceName(): Promise; + // (undocumented) + get secrets(): TaskSecrets | undefined; + // (undocumented) + get spec(): TaskSpec; +} + // @public export type TaskSecrets = { token: string | undefined; @@ -525,7 +525,7 @@ export class TaskWorker { // (undocumented) static createWorker(options: CreateWorkerOptions): TaskWorker; // (undocumented) - runOneTask(task: Task): Promise; + runOneTask(task: TaskContext): Promise; // (undocumented) start(): void; } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts index 0a843f4ddf..d07ff29fae 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -22,7 +22,7 @@ import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner'; import { TemplateActionRegistry } from '../actions'; import { ScmIntegrations } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; -import { Task, TaskSpec } from './types'; +import { TaskContext, TaskSpec } from './types'; describe('DefaultWorkflowRunner', () => { const logger = getVoidLogger(); @@ -38,7 +38,7 @@ describe('DefaultWorkflowRunner', () => { }), ); - const createMockTaskWithSpec = (spec: TaskSpec): Task => ({ + const createMockTaskWithSpec = (spec: TaskSpec): TaskContext => ({ spec, complete: async () => {}, done: false, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts index 831ef98784..77d0a7be09 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import { - Task, + TaskContext, WorkflowRunner, WorkflowResponse, TaskSpecV1beta2, @@ -72,7 +72,7 @@ export class HandlebarsWorkflowRunner implements WorkflowRunner { this.handlebars.registerHelper('eq', (a, b) => a === b); } - async execute(task: Task): Promise { + async execute(task: TaskContext): Promise { if (!isValidTaskSpec(task.spec)) { throw new InputError(`Task spec is not a valid v1beta2 task spec`); } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts index aa46747c68..91e772dc4e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts @@ -21,7 +21,7 @@ import { ScmIntegrations } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { HandlebarsWorkflowRunner } from './HandlebarsWorkflowRunner'; -import { Task, TaskSpec } from './types'; +import { TaskContext, TaskSpec } from './types'; import { RepoSpec } from '../actions/builtin/publish/util'; describe('LegacyWorkflowRunner', () => { @@ -37,7 +37,7 @@ describe('LegacyWorkflowRunner', () => { }), ); - const createMockTaskWithSpec = (spec: TaskSpec): Task => ({ + const createMockTaskWithSpec = (spec: TaskSpec): TaskContext => ({ spec, complete: async () => {}, done: false, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index 5cceb16324..587a93e1f2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -15,7 +15,7 @@ */ import { ScmIntegrations } from '@backstage/integration'; import { - Task, + TaskContext, TaskSpec, TaskSpecV1beta3, TaskStep, @@ -52,7 +52,13 @@ const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta3 => { return taskSpec.apiVersion === 'scaffolder.backstage.io/v1beta3'; }; -const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { +const createStepLogger = ({ + task, + step, +}: { + task: TaskContext; + step: TaskStep; +}) => { const metadata = { stepId: step.id }; const taskLogger = winston.createLogger({ level: process.env.LOG_LEVEL || 'info', @@ -162,7 +168,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { }); } - async execute(task: Task): Promise { + async execute(task: TaskContext): Promise { if (!isValidTaskSpec(task.spec)) { throw new InputError( 'Wrong template version executed with the workflow engine', diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts index c5d94c1324..76fb8e00f0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -17,7 +17,7 @@ import { getVoidLogger, DatabaseManager } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { DatabaseTaskStore } from './DatabaseTaskStore'; -import { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; +import { StorageTaskBroker, TaskManager } from './StorageTaskBroker'; import { TaskSecrets, TaskSpec, SerializedTaskEvent } from './types'; async function createStore(): Promise { @@ -48,7 +48,7 @@ describe('StorageTaskBroker', () => { it('should claim a dispatched work item', async () => { const broker = new StorageTaskBroker(storage, logger); await broker.dispatch({} as TaskSpec); - await expect(broker.claim()).resolves.toEqual(expect.any(TaskAgent)); + await expect(broker.claim()).resolves.toEqual(expect.any(TaskManager)); }); it('should wait for a dispatched work item', async () => { @@ -58,7 +58,7 @@ describe('StorageTaskBroker', () => { await expect(Promise.race([promise, 'waiting'])).resolves.toBe('waiting'); await broker.dispatch({} as TaskSpec); - await expect(promise).resolves.toEqual(expect.any(TaskAgent)); + await expect(promise).resolves.toEqual(expect.any(TaskManager)); }); it('should dispatch multiple items and claim them in order', async () => { @@ -70,9 +70,9 @@ describe('StorageTaskBroker', () => { const taskA = await broker.claim(); const taskB = await broker.claim(); const taskC = await broker.claim(); - await expect(taskA).toEqual(expect.any(TaskAgent)); - await expect(taskB).toEqual(expect.any(TaskAgent)); - await expect(taskC).toEqual(expect.any(TaskAgent)); + await expect(taskA).toEqual(expect.any(TaskManager)); + await expect(taskB).toEqual(expect.any(TaskManager)); + await expect(taskC).toEqual(expect.any(TaskManager)); await expect(taskA.spec.steps[0].id).toBe('a'); await expect(taskB.spec.steps[0].id).toBe('b'); await expect(taskC.spec.steps[0].id).toBe('c'); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 2f4387ca4a..7845ca994a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -18,7 +18,7 @@ import { assertError } from '@backstage/errors'; import { Logger } from 'winston'; import { CompletedTaskState, - Task, + TaskContext, TaskSecrets, TaskSpec, TaskStore, @@ -29,17 +29,17 @@ import { } from './types'; /** - * TaskAgent + * TaskManager * * @public */ -export class TaskAgent implements Task { +export class TaskManager implements TaskContext { private isDone = false; private heartbeatTimeoutId?: ReturnType; static create(state: TaskState, storage: TaskStore, logger: Logger) { - const agent = new TaskAgent(state, storage, logger); + const agent = new TaskManager(state, storage, logger); agent.startTimeout(); return agent; } @@ -135,11 +135,11 @@ export class StorageTaskBroker implements TaskBroker { ) {} private deferredDispatch = defer(); - async claim(): Promise { + async claim(): Promise { for (;;) { const pendingTask = await this.storage.claimTask(); if (pendingTask) { - return TaskAgent.create( + return TaskManager.create( { taskId: pendingTask.id, spec: pendingTask.spec, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index fcc215be99..d22bdc4e30 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Task, TaskBroker, WorkflowRunner } from './types'; +import { TaskContext, TaskBroker, WorkflowRunner } from './types'; import { HandlebarsWorkflowRunner } from './HandlebarsWorkflowRunner'; import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner'; import { Logger } from 'winston'; @@ -94,7 +94,7 @@ export class TaskWorker { })(); } - async runOneTask(task: Task) { + async runOneTask(task: TaskContext) { try { const { output } = task.spec.apiVersion === 'scaffolder.backstage.io/v1beta3' diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index 941c27ef0e..510c862dae 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -14,7 +14,7 @@ * limitations under the License. */ export { DatabaseTaskStore } from './DatabaseTaskStore'; -export { TaskAgent } from './StorageTaskBroker'; +export { TaskManager } from './StorageTaskBroker'; export type { TaskState } from './StorageTaskBroker'; export { TaskWorker } from './TaskWorker'; export type { CreateWorkerOptions } from './TaskWorker'; @@ -31,7 +31,7 @@ export type { Status, TaskEventType, TaskBroker, - Task, + TaskContext, TaskStore, DispatchResult, } from './types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index f2e13923ec..6b3075844d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -139,7 +139,7 @@ export type DispatchResult = { * * @public */ -export interface Task { +export interface TaskContext { spec: TaskSpec; secrets?: TaskSecrets; done: boolean; @@ -154,7 +154,7 @@ export interface Task { * @public */ export interface TaskBroker { - claim(): Promise; + claim(): Promise; dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; vacuumTasks(timeoutS: { timeoutS: number }): Promise; observe( @@ -221,5 +221,5 @@ export interface TaskStore { export type WorkflowResponse = { output: { [key: string]: JsonValue } }; export interface WorkflowRunner { - execute(task: Task): Promise; + execute(task: TaskContext): Promise; } From 2223b6de60bccd7e85048727762bfcdba08590d8 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 27 Oct 2021 10:54:32 +0100 Subject: [PATCH 11/16] adds a todo message to the router import Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/src/service/router.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 71b6d27b5e..1c5343d458 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -40,6 +40,12 @@ import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; +/** + * TODO: The following should import directly from the router file. + * Due to a circular dependency between this plugin and the + * plugin-scaffolder-backend-module-cookiecutter plugin, it results in an error: + * TypeError: _pluginscaffolderbackend.createTemplateAction is not a function + */ import { createRouter } from '../index'; const createCatalogClient = (templates: any[] = []) => From 3236071136feb402a2b6fa107677e90d4176860c Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 27 Oct 2021 12:46:47 +0100 Subject: [PATCH 12/16] change name of task worker builder function also makes it async Signed-off-by: Brian Fletcher --- .../src/scaffolder/tasks/TaskWorker.test.ts | 6 +++--- .../scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts | 2 +- plugins/scaffolder-backend/src/service/router.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index 3cb73abb85..f33b9182ce 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -82,7 +82,7 @@ describe('TaskWorker', () => { it('should call the legacy workflow runner when the apiVersion is not beta3', async () => { const broker = new StorageTaskBroker(storage, logger); - const taskWorker = TaskWorker.createWorker({ + const taskWorker = TaskWorker.create({ logger, workingDirectory, integrations, @@ -113,7 +113,7 @@ describe('TaskWorker', () => { it('should call the default workflow runner when the apiVersion is beta3', async () => { const broker = new StorageTaskBroker(storage, logger); - const taskWorker = TaskWorker.createWorker({ + const taskWorker = TaskWorker.create({ logger, workingDirectory, integrations, @@ -142,7 +142,7 @@ describe('TaskWorker', () => { }); const broker = new StorageTaskBroker(storage, logger); - const taskWorker = TaskWorker.createWorker({ + const taskWorker = TaskWorker.create({ logger, workingDirectory, integrations, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index d22bdc4e30..8d89735361 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -56,7 +56,7 @@ export type CreateWorkerOptions = { export class TaskWorker { private constructor(private readonly options: TaskWorkerOptions) {} - static createWorker(options: CreateWorkerOptions) { + static async create(options: CreateWorkerOptions): Promise { const { taskBroker, logger, diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 700f2c6e6f..231c998f8b 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -99,7 +99,7 @@ export async function createRouter( const workers = []; for (let i = 0; i < (taskWorkers || 1); i++) { - const worker = TaskWorker.createWorker({ + const worker = await TaskWorker.create({ taskBroker, actionRegistry, integrations, From ab6b7191c55a144dedf6b94448314bf4d672b882 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 27 Oct 2021 13:08:13 +0100 Subject: [PATCH 13/16] change test to await the task worker create Signed-off-by: Brian Fletcher --- .../src/scaffolder/tasks/TaskWorker.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index f33b9182ce..9b6baf3b84 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -82,7 +82,7 @@ describe('TaskWorker', () => { it('should call the legacy workflow runner when the apiVersion is not beta3', async () => { const broker = new StorageTaskBroker(storage, logger); - const taskWorker = TaskWorker.create({ + const taskWorker = await TaskWorker.create({ logger, workingDirectory, integrations, @@ -113,7 +113,7 @@ describe('TaskWorker', () => { it('should call the default workflow runner when the apiVersion is beta3', async () => { const broker = new StorageTaskBroker(storage, logger); - const taskWorker = TaskWorker.create({ + const taskWorker = await TaskWorker.create({ logger, workingDirectory, integrations, @@ -142,7 +142,7 @@ describe('TaskWorker', () => { }); const broker = new StorageTaskBroker(storage, logger); - const taskWorker = TaskWorker.create({ + const taskWorker = await TaskWorker.create({ logger, workingDirectory, integrations, From 93d0ea461258ffe09070ef7088cbf9baf2bee0bd Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 27 Oct 2021 13:18:12 +0100 Subject: [PATCH 14/16] add api reports changes Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/api-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 5441c4de6b..6f9822cd09 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -523,7 +523,7 @@ export type TaskStoreListEventsOptions = { // @public export class TaskWorker { // (undocumented) - static createWorker(options: CreateWorkerOptions): TaskWorker; + static create(options: CreateWorkerOptions): Promise; // (undocumented) runOneTask(task: TaskContext): Promise; // (undocumented) From 66b5024b7c1eb4a944d79271a30d007405941c08 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 28 Oct 2021 13:02:07 +0100 Subject: [PATCH 15/16] only create datastore if needed for the broker Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/src/service/router.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 231c998f8b..814b235bf2 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -89,12 +89,17 @@ export async function createRouter( const workingDirectory = await getWorkingDirectory(config, logger); const entityClient = new CatalogEntityClient(catalogClient); const integrations = ScmIntegrations.fromConfig(config); + let taskBroker: TaskBroker; + + if (!options.taskBroker) { + const databaseTaskStore = await DatabaseTaskStore.create({ + database: await database.getClient(), + }); + taskBroker = new StorageTaskBroker(databaseTaskStore, logger); + } else { + taskBroker = options.taskBroker; + } - const databaseTaskStore = await DatabaseTaskStore.create({ - database: await database.getClient(), - }); - const taskBroker = - options.taskBroker || new StorageTaskBroker(databaseTaskStore, logger); const actionRegistry = new TemplateActionRegistry(); const workers = []; From 8796c00de828f81c478f2fcf0a66b6530a00804a Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 28 Oct 2021 15:09:51 +0100 Subject: [PATCH 16/16] change return object for broker observe function Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/api-report.md | 4 +++- .../src/scaffolder/tasks/StorageTaskBroker.ts | 4 ++-- plugins/scaffolder-backend/src/scaffolder/tasks/types.ts | 2 +- plugins/scaffolder-backend/src/service/router.ts | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 6f9822cd09..3f9b77a3a3 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -369,7 +369,9 @@ export interface TaskBroker { events: SerializedTaskEvent[]; }, ) => void, - ): () => void; + ): { + unsubscribe: () => void; + }; // (undocumented) vacuumTasks(timeoutS: { timeoutS: number }): Promise; } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 7845ca994a..7e7ebe5436 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -178,7 +178,7 @@ export class StorageTaskBroker implements TaskBroker { error: Error | undefined, result: { events: SerializedTaskEvent[] }, ) => void, - ): () => void { + ): { unsubscribe: () => void } { const { taskId } = options; let cancelled = false; @@ -205,7 +205,7 @@ export class StorageTaskBroker implements TaskBroker { } })(); - return unsubscribe; + return { unsubscribe }; } async vacuumTasks(timeoutS: { timeoutS: number }): Promise { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 6b3075844d..6cfd914bd2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -166,7 +166,7 @@ export interface TaskBroker { error: Error | undefined, result: { events: SerializedTaskEvent[] }, ) => void, - ): () => void; + ): { unsubscribe: () => void }; get(taskId: string): Promise; } diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 814b235bf2..a78b04da78 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -258,7 +258,7 @@ export async function createRouter( }); // After client opens connection send all events as string - const unsubscribe = taskBroker.observe( + const { unsubscribe } = taskBroker.observe( { taskId, after }, (error, { events }) => { if (error) {