From 9990df8a1f5811e2c0f1163b685cfa1f2d46ba63 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 14 Oct 2021 17:38:51 +0100 Subject: [PATCH] Exports required to implement scaffolder broker Signed-off-by: Brian Fletcher --- .changeset/lovely-cars-sneeze.md | 5 + plugins/scaffolder-backend/api-report.md | 309 ++++++++++++++++++ .../src/scaffolder/index.ts | 24 ++ .../src/scaffolder/tasks/DatabaseTaskStore.ts | 8 + .../scaffolder/tasks/DefaultWorkflowRunner.ts | 4 + .../scaffolder/tasks/LegacyWorkflowRunner.ts | 4 + .../src/scaffolder/tasks/StorageTaskBroker.ts | 11 +- .../src/scaffolder/tasks/TaskWorker.ts | 4 + .../src/scaffolder/tasks/index.ts | 4 +- .../src/scaffolder/tasks/types.ts | 65 +++- .../scaffolder-backend/src/service/router.ts | 16 +- 11 files changed, 445 insertions(+), 9 deletions(-) create mode 100644 .changeset/lovely-cars-sneeze.md diff --git a/.changeset/lovely-cars-sneeze.md b/.changeset/lovely-cars-sneeze.md new file mode 100644 index 0000000000..8e6bfd691c --- /dev/null +++ b/.changeset/lovely-cars-sneeze.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Expose some classes and interfaces public so TaskWorkers can run externally from the scaffolder API. diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index ef4defcc50..e0e7ad7ee5 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -16,6 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import express from 'express'; import { JsonObject } from '@backstage/config'; import { JsonValue } from '@backstage/config'; +import { Knex } from 'knex'; import { LocationSpec } from '@backstage/catalog-model'; import { Logger as Logger_2 } from 'winston'; import { Octokit } from '@octokit/rest'; @@ -25,6 +26,7 @@ import { ScmIntegrationRegistry } from '@backstage/integration'; import { ScmIntegrations } from '@backstage/integration'; import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; import { UrlReader } from '@backstage/backend-common'; +import * as winston from 'winston'; import { Writable } from 'stream'; // Warning: (ae-forgotten-export) The symbol "InputBase" needs to be exported by the entry point index.d.ts @@ -189,6 +191,78 @@ export const createTemplateAction: < templateAction: TemplateAction, ) => TemplateAction; +// @public +export class DatabaseTaskStore implements TaskStore { + constructor(db: Knex); + // (undocumented) + claimTask(): Promise; + // (undocumented) + completeTask({ + taskId, + status, + eventBody, + }: { + taskId: string; + status: Status; + eventBody: JsonObject; + }): Promise; + // (undocumented) + static create(knex: Knex): Promise; + // (undocumented) + createTask( + spec: TaskSpec, + secrets?: TaskSecrets, + ): Promise<{ + taskId: string; + }>; + // (undocumented) + emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise; + // (undocumented) + getTask(taskId: string): Promise; + // (undocumented) + heartbeatTask(taskId: string): Promise; + // Warning: (ae-forgotten-export) The symbol "TaskStoreGetEventsOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + listEvents({ taskId, after }: TaskStoreGetEventsOptions): Promise<{ + events: DbTaskEventRow[]; + }>; + // (undocumented) + listStaleTasks({ timeoutS }: { timeoutS: number }): Promise<{ + tasks: { + taskId: string; + }[]; + }>; +} + +// @public +export type DbTaskRow = { + id: string; + spec: TaskSpec; + status: Status; + createdAt: string; + lastHeartbeatAt?: string; + secrets?: TaskSecrets; +}; + +// Warning: (ae-forgotten-export) The symbol "WorkflowRunner" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "DefaultWorkflowRunner" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class DefaultWorkflowRunner implements WorkflowRunner { + // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts + constructor(options: Options_3); + // Warning: (ae-forgotten-export) The symbol "WorkflowResponse" needs to be exported by the entry point index.d.ts + // + // (undocumented) + execute(task: Task): Promise; +} + +// @public +export type DispatchResult = { + taskId: string; +}; + // Warning: (ae-missing-release-tag) "fetchContents" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -206,6 +280,14 @@ export function fetchContents({ outputPath: string; }): Promise; +// @public +export class LegacyWorkflowRunner implements WorkflowRunner { + // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts + constructor(options: Options_2); + // (undocumented) + execute(task: Task): Promise; +} + // Warning: (ae-missing-release-tag) "OctokitProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -216,6 +298,15 @@ export class OctokitProvider { getOctokit(repoUrl: string): Promise; } +// @public +export type RawDbTaskEventRow = { + id: number; + task_id: string; + body: string; + event_type: TaskEventType; + created_at: string; +}; + // Warning: (ae-missing-release-tag) "RouterOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -235,6 +326,8 @@ export interface RouterOptions { // (undocumented) reader: UrlReader; // (undocumented) + taskBroker?: TaskBroker; + // (undocumented) taskWorkers?: number; } @@ -260,6 +353,218 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { validateEntityKind(entity: Entity): Promise; } +// @public +export type Status = + | 'open' + | 'processing' + | 'failed' + | 'cancelled' + | 'completed'; + +// @public +export class StorageTaskBroker implements TaskBroker { + constructor(storage: TaskStore, logger: Logger_2); + // (undocumented) + claim(): Promise; + // (undocumented) + dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; + // (undocumented) + get(taskId: string): Promise; + // (undocumented) + protected readonly logger: Logger_2; + // (undocumented) + observe( + options: { + taskId: string; + after: number | undefined; + }, + callback: ( + error: Error | undefined, + result: { + events: DbTaskEventRow[]; + }, + ) => void, + ): () => void; + // (undocumented) + protected readonly storage: TaskStore; + // (undocumented) + vacuumTasks(timeoutS: { timeoutS: number }): Promise; +} + +// @public +export interface Task { + // Warning: (ae-forgotten-export) The symbol "CompletedTaskState" needs to be exported by the entry point index.d.ts + // + // (undocumented) + complete(result: CompletedTaskState, metadata?: JsonValue): Promise; + // (undocumented) + done: boolean; + // (undocumented) + emitLog(message: string, metadata?: JsonValue): Promise; + // (undocumented) + getWorkspaceName(): Promise; + // (undocumented) + secrets?: TaskSecrets; + // (undocumented) + spec: TaskSpec; +} + +// Warning: (ae-missing-release-tag) "TaskAgent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class TaskAgent implements Task { + // (undocumented) + complete(result: CompletedTaskState, metadata?: JsonObject): Promise; + // Warning: (ae-forgotten-export) The symbol "TaskState" needs to be exported by the entry point index.d.ts + // + // (undocumented) + static create( + state: TaskState, + storage: TaskStore, + logger: Logger_2, + ): TaskAgent; + // (undocumented) + get done(): boolean; + // (undocumented) + emitLog(message: string, metadata?: JsonObject): Promise; + // (undocumented) + getWorkspaceName(): Promise; + // (undocumented) + get secrets(): TaskSecrets | undefined; + // (undocumented) + get spec(): TaskSpec; +} + +// @public +export interface TaskBroker { + // (undocumented) + claim(): Promise; + // (undocumented) + dispatch( + spec: TaskSpec, + secretTaskSecretss?: TaskSecrets, + ): Promise; + // (undocumented) + get(taskId: string): Promise; + // (undocumented) + observe( + options: { + taskId: string; + after: number | undefined; + }, + callback: ( + error: Error | undefined, + result: { + events: DbTaskEventRow[]; + }, + ) => void, + ): () => void; + // (undocumented) + vacuumTasks(timeoutS: { timeoutS: number }): Promise; +} + +// @public +export type TaskEventType = 'completion' | 'log'; + +// @public +export type TaskSecrets = { + token: string | undefined; +}; + +// @public +export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3; + +// @public +export interface TaskSpecV1beta2 { + // (undocumented) + apiVersion: 'backstage.io/v1beta2'; + // (undocumented) + baseUrl?: string; + // (undocumented) + output: { + [name: string]: string; + }; + // (undocumented) + steps: Array<{ + id: string; + name: string; + action: string; + input?: JsonObject; + if?: string | boolean; + }>; + // (undocumented) + values: JsonObject; +} + +// @public +export interface TaskSpecV1beta3 { + // (undocumented) + apiVersion: 'scaffolder.backstage.io/v1beta3'; + // (undocumented) + baseUrl?: string; + // (undocumented) + output: { + [name: string]: JsonValue; + }; + // (undocumented) + parameters: JsonObject; + // Warning: (ae-forgotten-export) The symbol "TaskStep" needs to be exported by the entry point index.d.ts + // + // (undocumented) + steps: TaskStep[]; +} + +// @public +export interface TaskStore { + // (undocumented) + claimTask(): Promise; + // (undocumented) + completeTask(options: { + taskId: string; + status: Status; + eventBody: JsonObject; + }): Promise; + // (undocumented) + createTask( + task: TaskSpec, + secrets?: TaskSecrets, + ): Promise<{ + taskId: string; + }>; + // (undocumented) + emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise; + // (undocumented) + getTask(taskId: string): Promise; + // (undocumented) + heartbeatTask(taskId: string): Promise; + // (undocumented) + listEvents({ taskId, after }: TaskStoreGetEventsOptions): Promise<{ + events: DbTaskEventRow[]; + }>; + // (undocumented) + listStaleTasks(options: { timeoutS: number }): Promise<{ + tasks: { + taskId: string; + }[]; + }>; +} + +// @public +export type TaskStoreEmitOptions = { + taskId: string; + body: JsonObject; +}; + +// @public +export class TaskWorker { + // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts + constructor(options: Options); + // (undocumented) + runOneTask(task: Task): Promise; + // (undocumented) + start(): void; +} + // Warning: (ae-missing-release-tag) "TemplateAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -286,4 +591,8 @@ export class TemplateActionRegistry { action: TemplateAction, ): void; } + +// Warnings were encountered during analysis: +// +// src/scaffolder/tasks/DatabaseTaskStore.d.ts:51:9 - (ae-forgotten-export) The symbol "DbTaskEventRow" needs to be exported by the entry point index.d.ts ``` diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index 284e372b4c..cf2a554189 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -14,3 +14,27 @@ * limitations under the License. */ export * from './actions'; +export { + TaskAgent, + DatabaseTaskStore, + StorageTaskBroker, + TaskWorker, + LegacyWorkflowRunner, + DefaultWorkflowRunner, +} from './tasks'; +export type { + Task, + TaskBroker, + TaskSpec, + TaskSecrets, + DispatchResult, + TaskStore, + TaskStoreEmitOptions, + DbTaskRow, + TaskSpecV1beta2, + TaskSpecV1beta3, + Status, + TaskEventType, +} from './tasks/types'; + +export type { RawDbTaskEventRow } from './tasks/DatabaseTaskStore'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 09cba596b1..8f659cad02 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -46,6 +46,10 @@ export type RawDbTaskRow = { secrets?: string; }; +/** + * RawDbTaskEventRow + * @public + */ export type RawDbTaskEventRow = { id: number; task_id: string; @@ -54,6 +58,10 @@ export type RawDbTaskEventRow = { created_at: string; }; +/** + * DatabaseTaskStore + * @public + */ export class DatabaseTaskStore implements TaskStore { static async create(knex: Knex): Promise { await knex.migrate.latest({ diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index 2ac309a1d4..7b2b437be2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -77,6 +77,10 @@ const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { return { taskLogger, streamLogger }; }; +/* + * DefaultWorkflowRunner + * @public + */ export class DefaultWorkflowRunner implements WorkflowRunner { private readonly nunjucks: nunjucks.Environment; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts index 97e3a4dec4..e71fe596cd 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts @@ -43,9 +43,13 @@ type Options = { const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta2 => taskSpec.apiVersion === 'backstage.io/v1beta2'; + /** * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced * with the default workflow runner interface in the future so this entire thing can go bye bye. + * + * LegacyWorkflowRunner + * @public */ export class LegacyWorkflowRunner implements WorkflowRunner { private readonly handlebars: typeof Handlebars; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 867e5bb127..59f605702d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -27,6 +27,9 @@ import { DbTaskRow, } from './types'; +/* + * @public + */ export class TaskAgent implements Task { private isDone = false; @@ -117,10 +120,14 @@ function defer() { return { promise, resolve }; } +/** + * StorageTaskBroker + * @public + */ export class StorageTaskBroker implements TaskBroker { constructor( - private readonly storage: TaskStore, - private readonly logger: Logger, + protected readonly storage: TaskStore, + protected readonly logger: Logger, ) {} private deferredDispatch = defer(); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 7122c2fc3e..1f6d9d3381 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -24,6 +24,10 @@ type Options = { }; }; +/** + * TaskWorker + * @public + */ export class TaskWorker { constructor(private readonly options: Options) {} start() { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index dd13264aed..c068e8d7e1 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -15,5 +15,7 @@ */ export { DatabaseTaskStore } from './DatabaseTaskStore'; -export { StorageTaskBroker } from './StorageTaskBroker'; +export { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; export { TaskWorker } from './TaskWorker'; +export { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; +export { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index cd93404165..63b409e7ca 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -16,6 +16,10 @@ import { JsonValue, JsonObject } from '@backstage/config'; +/** + * Status + * @public + */ export type Status = | 'open' | 'processing' @@ -25,6 +29,10 @@ export type Status = export type CompletedTaskState = 'failed' | 'completed'; +/** + * DbTaskRow + * @public + */ export type DbTaskRow = { id: string; spec: TaskSpec; @@ -34,7 +42,16 @@ export type DbTaskRow = { secrets?: TaskSecrets; }; +/** + * TaskEventType + * @public + */ export type TaskEventType = 'completion' | 'log'; + +/** + * DbTaskEventRow + * @public + */ export type DbTaskEventRow = { id: number; taskId: string; @@ -43,6 +60,10 @@ export type DbTaskEventRow = { createdAt: string; }; +/** + * TaskSpecV1beta2 + * @public + */ export interface TaskSpecV1beta2 { apiVersion: 'backstage.io/v1beta2'; baseUrl?: string; @@ -64,6 +85,11 @@ export interface TaskStep { input?: JsonObject; if?: string | boolean; } + +/** + * TaskSpecV1beta3 + * @public + */ export interface TaskSpecV1beta3 { apiVersion: 'scaffolder.backstage.io/v1beta3'; baseUrl?: string; @@ -72,16 +98,32 @@ export interface TaskSpecV1beta3 { output: { [name: string]: JsonValue }; } +/** + * TaskSpec + * @public + */ export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3; +/** + * TaskSecrets + * @public + */ export type TaskSecrets = { token: string | undefined; }; +/** + * DispatchResult + * @public + */ export type DispatchResult = { taskId: string; }; +/** + * Task + * @public + */ export interface Task { spec: TaskSpec; secrets?: TaskSecrets; @@ -91,9 +133,16 @@ export interface Task { getWorkspaceName(): Promise; } +/** + * TaskBroker + * @public + */ export interface TaskBroker { claim(): Promise; - dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; + dispatch( + spec: TaskSpec, + secretTaskSecretss?: TaskSecrets, + ): Promise; vacuumTasks(timeoutS: { timeoutS: number }): Promise; observe( options: { @@ -105,8 +154,14 @@ export interface TaskBroker { result: { events: DbTaskEventRow[] }, ) => void, ): () => void; + + get(taskId: string): Promise; } +/** + * TaskStoreEmitOptions + * @public + */ export type TaskStoreEmitOptions = { taskId: string; body: JsonObject; @@ -117,6 +172,10 @@ export type TaskStoreGetEventsOptions = { after?: number | undefined; }; +/** + * TaskStore + * @public + */ export interface TaskStore { createTask( task: TaskSpec, @@ -142,6 +201,10 @@ export interface TaskStore { } export type WorkflowResponse = { output: { [key: string]: JsonValue } }; +/* + * WorkflowRunner + * @public + */ export interface WorkflowRunner { execute(task: Task): Promise; } diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index f8ba99336f..5f855a6e88 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -25,7 +25,13 @@ import { StorageTaskBroker, TaskWorker, } from '../scaffolder/tasks'; -import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry'; +import { + TemplateActionRegistry, + TemplateAction, + createBuiltinActions, + TaskBroker, + TaskSpec, +} from '../scaffolder'; import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { ContainerRunner, @@ -38,11 +44,9 @@ import { TemplateEntityV1beta2, Entity } from '@backstage/catalog-model'; import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; import { ScmIntegrations } from '@backstage/integration'; -import { TemplateAction } from '../scaffolder/actions'; -import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; + import { LegacyWorkflowRunner } from '../scaffolder/tasks/LegacyWorkflowRunner'; import { DefaultWorkflowRunner } from '../scaffolder/tasks/DefaultWorkflowRunner'; -import { TaskSpec } from '../scaffolder/tasks/types'; export interface RouterOptions { logger: Logger; @@ -53,6 +57,7 @@ export interface RouterOptions { actions?: TemplateAction[]; taskWorkers?: number; containerRunner: ContainerRunner; + taskBroker?: TaskBroker; } function isSupportedTemplate( @@ -89,7 +94,8 @@ export async function createRouter( const databaseTaskStore = await DatabaseTaskStore.create( await database.getClient(), ); - const taskBroker = new StorageTaskBroker(databaseTaskStore, logger); + const taskBroker = + options.taskBroker || new StorageTaskBroker(databaseTaskStore, logger); const actionRegistry = new TemplateActionRegistry(); const legacyWorkflowRunner = new LegacyWorkflowRunner({ logger,