Exports required to implement scaffolder broker

Signed-off-by: Brian Fletcher <brian@roadie.io>

.changeset/lovely-cars-sneeze.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Expose some classes and interfaces public so TaskWorkers can run externally from the scaffolder API.

plugins/scaffolder-backend/api-report.md

@@ -16,6 +16,7 @@ import { Entity } from '@backstage/catalog-model';
 import express from 'express';
 import { JsonObject } from '@backstage/config';
 import { JsonValue } from '@backstage/config';
+import { Knex } from 'knex';
 import { LocationSpec } from '@backstage/catalog-model';
 import { Logger as Logger_2 } from 'winston';
 import { Octokit } from '@octokit/rest';
@@ -25,6 +26,7 @@ import { ScmIntegrationRegistry } from '@backstage/integration';
 import { ScmIntegrations } from '@backstage/integration';
 import { TemplateEntityV1beta2 } from '@backstage/catalog-model';
 import { UrlReader } from '@backstage/backend-common';
+import * as winston from 'winston';
 import { Writable } from 'stream';
 
 // Warning: (ae-forgotten-export) The symbol "InputBase" needs to be exported by the entry point index.d.ts
@@ -189,6 +191,78 @@ export const createTemplateAction: <
   templateAction: TemplateAction<Input>,
 ) => TemplateAction<any>;
 
+// @public
+export class DatabaseTaskStore implements TaskStore {
+  constructor(db: Knex);
+  // (undocumented)
+  claimTask(): Promise<DbTaskRow | undefined>;
+  // (undocumented)
+  completeTask({
+    taskId,
+    status,
+    eventBody,
+  }: {
+    taskId: string;
+    status: Status;
+    eventBody: JsonObject;
+  }): Promise<void>;
+  // (undocumented)
+  static create(knex: Knex): Promise<DatabaseTaskStore>;
+  // (undocumented)
+  createTask(
+    spec: TaskSpec,
+    secrets?: TaskSecrets,
+  ): Promise<{
+    taskId: string;
+  }>;
+  // (undocumented)
+  emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise<void>;
+  // (undocumented)
+  getTask(taskId: string): Promise<DbTaskRow>;
+  // (undocumented)
+  heartbeatTask(taskId: string): Promise<void>;
+  // Warning: (ae-forgotten-export) The symbol "TaskStoreGetEventsOptions" needs to be exported by the entry point index.d.ts
+  //
+  // (undocumented)
+  listEvents({ taskId, after }: TaskStoreGetEventsOptions): Promise<{
+    events: DbTaskEventRow[];
+  }>;
+  // (undocumented)
+  listStaleTasks({ timeoutS }: { timeoutS: number }): Promise<{
+    tasks: {
+      taskId: string;
+    }[];
+  }>;
+}
+
+// @public
+export type DbTaskRow = {
+  id: string;
+  spec: TaskSpec;
+  status: Status;
+  createdAt: string;
+  lastHeartbeatAt?: string;
+  secrets?: TaskSecrets;
+};
+
+// Warning: (ae-forgotten-export) The symbol "WorkflowRunner" needs to be exported by the entry point index.d.ts
+// Warning: (ae-missing-release-tag) "DefaultWorkflowRunner" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class DefaultWorkflowRunner implements WorkflowRunner {
+  // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts
+  constructor(options: Options_3);
+  // Warning: (ae-forgotten-export) The symbol "WorkflowResponse" needs to be exported by the entry point index.d.ts
+  //
+  // (undocumented)
+  execute(task: Task): Promise<WorkflowResponse>;
+}
+
+// @public
+export type DispatchResult = {
+  taskId: string;
+};
+
 // Warning: (ae-missing-release-tag) "fetchContents" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -206,6 +280,14 @@ export function fetchContents({
   outputPath: string;
 }): Promise<void>;
 
+// @public
+export class LegacyWorkflowRunner implements WorkflowRunner {
+  // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts
+  constructor(options: Options_2);
+  // (undocumented)
+  execute(task: Task): Promise<WorkflowResponse>;
+}
+
 // Warning: (ae-missing-release-tag) "OctokitProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public
@@ -216,6 +298,15 @@ export class OctokitProvider {
   getOctokit(repoUrl: string): Promise<OctokitIntegration>;
 }
 
+// @public
+export type RawDbTaskEventRow = {
+  id: number;
+  task_id: string;
+  body: string;
+  event_type: TaskEventType;
+  created_at: string;
+};
+
 // Warning: (ae-missing-release-tag) "RouterOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -235,6 +326,8 @@ export interface RouterOptions {
   // (undocumented)
   reader: UrlReader;
   // (undocumented)
+  taskBroker?: TaskBroker;
+  // (undocumented)
   taskWorkers?: number;
 }
 
@@ -260,6 +353,218 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor {
   validateEntityKind(entity: Entity): Promise<boolean>;
 }
 
+// @public
+export type Status =
+  | 'open'
+  | 'processing'
+  | 'failed'
+  | 'cancelled'
+  | 'completed';
+
+// @public
+export class StorageTaskBroker implements TaskBroker {
+  constructor(storage: TaskStore, logger: Logger_2);
+  // (undocumented)
+  claim(): Promise<Task>;
+  // (undocumented)
+  dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise<DispatchResult>;
+  // (undocumented)
+  get(taskId: string): Promise<DbTaskRow>;
+  // (undocumented)
+  protected readonly logger: Logger_2;
+  // (undocumented)
+  observe(
+    options: {
+      taskId: string;
+      after: number | undefined;
+    },
+    callback: (
+      error: Error | undefined,
+      result: {
+        events: DbTaskEventRow[];
+      },
+    ) => void,
+  ): () => void;
+  // (undocumented)
+  protected readonly storage: TaskStore;
+  // (undocumented)
+  vacuumTasks(timeoutS: { timeoutS: number }): Promise<void>;
+}
+
+// @public
+export interface Task {
+  // Warning: (ae-forgotten-export) The symbol "CompletedTaskState" needs to be exported by the entry point index.d.ts
+  //
+  // (undocumented)
+  complete(result: CompletedTaskState, metadata?: JsonValue): Promise<void>;
+  // (undocumented)
+  done: boolean;
+  // (undocumented)
+  emitLog(message: string, metadata?: JsonValue): Promise<void>;
+  // (undocumented)
+  getWorkspaceName(): Promise<string>;
+  // (undocumented)
+  secrets?: TaskSecrets;
+  // (undocumented)
+  spec: TaskSpec;
+}
+
+// Warning: (ae-missing-release-tag) "TaskAgent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class TaskAgent implements Task {
+  // (undocumented)
+  complete(result: CompletedTaskState, metadata?: JsonObject): Promise<void>;
+  // Warning: (ae-forgotten-export) The symbol "TaskState" needs to be exported by the entry point index.d.ts
+  //
+  // (undocumented)
+  static create(
+    state: TaskState,
+    storage: TaskStore,
+    logger: Logger_2,
+  ): TaskAgent;
+  // (undocumented)
+  get done(): boolean;
+  // (undocumented)
+  emitLog(message: string, metadata?: JsonObject): Promise<void>;
+  // (undocumented)
+  getWorkspaceName(): Promise<string>;
+  // (undocumented)
+  get secrets(): TaskSecrets | undefined;
+  // (undocumented)
+  get spec(): TaskSpec;
+}
+
+// @public
+export interface TaskBroker {
+  // (undocumented)
+  claim(): Promise<Task>;
+  // (undocumented)
+  dispatch(
+    spec: TaskSpec,
+    secretTaskSecretss?: TaskSecrets,
+  ): Promise<DispatchResult>;
+  // (undocumented)
+  get(taskId: string): Promise<DbTaskRow>;
+  // (undocumented)
+  observe(
+    options: {
+      taskId: string;
+      after: number | undefined;
+    },
+    callback: (
+      error: Error | undefined,
+      result: {
+        events: DbTaskEventRow[];
+      },
+    ) => void,
+  ): () => void;
+  // (undocumented)
+  vacuumTasks(timeoutS: { timeoutS: number }): Promise<void>;
+}
+
+// @public
+export type TaskEventType = 'completion' | 'log';
+
+// @public
+export type TaskSecrets = {
+  token: string | undefined;
+};
+
+// @public
+export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3;
+
+// @public
+export interface TaskSpecV1beta2 {
+  // (undocumented)
+  apiVersion: 'backstage.io/v1beta2';
+  // (undocumented)
+  baseUrl?: string;
+  // (undocumented)
+  output: {
+    [name: string]: string;
+  };
+  // (undocumented)
+  steps: Array<{
+    id: string;
+    name: string;
+    action: string;
+    input?: JsonObject;
+    if?: string | boolean;
+  }>;
+  // (undocumented)
+  values: JsonObject;
+}
+
+// @public
+export interface TaskSpecV1beta3 {
+  // (undocumented)
+  apiVersion: 'scaffolder.backstage.io/v1beta3';
+  // (undocumented)
+  baseUrl?: string;
+  // (undocumented)
+  output: {
+    [name: string]: JsonValue;
+  };
+  // (undocumented)
+  parameters: JsonObject;
+  // Warning: (ae-forgotten-export) The symbol "TaskStep" needs to be exported by the entry point index.d.ts
+  //
+  // (undocumented)
+  steps: TaskStep[];
+}
+
+// @public
+export interface TaskStore {
+  // (undocumented)
+  claimTask(): Promise<DbTaskRow | undefined>;
+  // (undocumented)
+  completeTask(options: {
+    taskId: string;
+    status: Status;
+    eventBody: JsonObject;
+  }): Promise<void>;
+  // (undocumented)
+  createTask(
+    task: TaskSpec,
+    secrets?: TaskSecrets,
+  ): Promise<{
+    taskId: string;
+  }>;
+  // (undocumented)
+  emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise<void>;
+  // (undocumented)
+  getTask(taskId: string): Promise<DbTaskRow>;
+  // (undocumented)
+  heartbeatTask(taskId: string): Promise<void>;
+  // (undocumented)
+  listEvents({ taskId, after }: TaskStoreGetEventsOptions): Promise<{
+    events: DbTaskEventRow[];
+  }>;
+  // (undocumented)
+  listStaleTasks(options: { timeoutS: number }): Promise<{
+    tasks: {
+      taskId: string;
+    }[];
+  }>;
+}
+
+// @public
+export type TaskStoreEmitOptions = {
+  taskId: string;
+  body: JsonObject;
+};
+
+// @public
+export class TaskWorker {
+  // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts
+  constructor(options: Options);
+  // (undocumented)
+  runOneTask(task: Task): Promise<void>;
+  // (undocumented)
+  start(): void;
+}
+
 // Warning: (ae-missing-release-tag) "TemplateAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -286,4 +591,8 @@ export class TemplateActionRegistry {
     action: TemplateAction<Parameters>,
   ): void;
 }
+
+// Warnings were encountered during analysis:
+//
+// src/scaffolder/tasks/DatabaseTaskStore.d.ts:51:9 - (ae-forgotten-export) The symbol "DbTaskEventRow" needs to be exported by the entry point index.d.ts
 ```

plugins/scaffolder-backend/src/scaffolder/index.ts

@@ -14,3 +14,27 @@
  * limitations under the License.
  */
 export * from './actions';
+export {
+  TaskAgent,
+  DatabaseTaskStore,
+  StorageTaskBroker,
+  TaskWorker,
+  LegacyWorkflowRunner,
+  DefaultWorkflowRunner,
+} from './tasks';
+export type {
+  Task,
+  TaskBroker,
+  TaskSpec,
+  TaskSecrets,
+  DispatchResult,
+  TaskStore,
+  TaskStoreEmitOptions,
+  DbTaskRow,
+  TaskSpecV1beta2,
+  TaskSpecV1beta3,
+  Status,
+  TaskEventType,
+} from './tasks/types';
+
+export type { RawDbTaskEventRow } from './tasks/DatabaseTaskStore';

plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts

@@ -46,6 +46,10 @@ export type RawDbTaskRow = {
   secrets?: string;
 };
 
+/**
+ * RawDbTaskEventRow
+ * @public
+ */
 export type RawDbTaskEventRow = {
   id: number;
   task_id: string;
@@ -54,6 +58,10 @@ export type RawDbTaskEventRow = {
   created_at: string;
 };
 
+/**
+ * DatabaseTaskStore
+ * @public
+ */
 export class DatabaseTaskStore implements TaskStore {
   static async create(knex: Knex): Promise<DatabaseTaskStore> {
     await knex.migrate.latest({

plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts

@@ -77,6 +77,10 @@ const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => {
   return { taskLogger, streamLogger };
 };
 
+/*
+ * DefaultWorkflowRunner
+ * @public
+ */
 export class DefaultWorkflowRunner implements WorkflowRunner {
   private readonly nunjucks: nunjucks.Environment;
 

plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts

@@ -43,9 +43,13 @@ type Options = {
 
 const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta2 =>
   taskSpec.apiVersion === 'backstage.io/v1beta2';
+
 /**
  * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced
  * with the default workflow runner interface in the future so this entire thing can go bye bye.
+ *
+ * LegacyWorkflowRunner
+ * @public
  */
 export class LegacyWorkflowRunner implements WorkflowRunner {
   private readonly handlebars: typeof Handlebars;

plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts

@@ -27,6 +27,9 @@ import {
   DbTaskRow,
 } from './types';
 
+/*
+ * @public
+ */
 export class TaskAgent implements Task {
   private isDone = false;
 
@@ -117,10 +120,14 @@ function defer() {
   return { promise, resolve };
 }
 
+/**
+ * StorageTaskBroker
+ * @public
+ */
 export class StorageTaskBroker implements TaskBroker {
   constructor(
-    private readonly storage: TaskStore,
-    private readonly logger: Logger,
+    protected readonly storage: TaskStore,
+    protected readonly logger: Logger,
   ) {}
   private deferredDispatch = defer();
 

plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts

@@ -24,6 +24,10 @@ type Options = {
   };
 };
 
+/**
+ * TaskWorker
+ * @public
+ */
 export class TaskWorker {
   constructor(private readonly options: Options) {}
   start() {

plugins/scaffolder-backend/src/scaffolder/tasks/index.ts

@@ -15,5 +15,7 @@
  */
 
 export { DatabaseTaskStore } from './DatabaseTaskStore';
-export { StorageTaskBroker } from './StorageTaskBroker';
+export { StorageTaskBroker, TaskAgent } from './StorageTaskBroker';
 export { TaskWorker } from './TaskWorker';
+export { LegacyWorkflowRunner } from './LegacyWorkflowRunner';
+export { DefaultWorkflowRunner } from './DefaultWorkflowRunner';

plugins/scaffolder-backend/src/scaffolder/tasks/types.ts

@@ -16,6 +16,10 @@
 
 import { JsonValue, JsonObject } from '@backstage/config';
 
+/**
+ * Status
+ * @public
+ */
 export type Status =
   | 'open'
   | 'processing'
@@ -25,6 +29,10 @@ export type Status =
 
 export type CompletedTaskState = 'failed' | 'completed';
 
+/**
+ * DbTaskRow
+ * @public
+ */
 export type DbTaskRow = {
   id: string;
   spec: TaskSpec;
@@ -34,7 +42,16 @@ export type DbTaskRow = {
   secrets?: TaskSecrets;
 };
 
+/**
+ * TaskEventType
+ * @public
+ */
 export type TaskEventType = 'completion' | 'log';
+
+/**
+ * DbTaskEventRow
+ * @public
+ */
 export type DbTaskEventRow = {
   id: number;
   taskId: string;
@@ -43,6 +60,10 @@ export type DbTaskEventRow = {
   createdAt: string;
 };
 
+/**
+ * TaskSpecV1beta2
+ * @public
+ */
 export interface TaskSpecV1beta2 {
   apiVersion: 'backstage.io/v1beta2';
   baseUrl?: string;
@@ -64,6 +85,11 @@ export interface TaskStep {
   input?: JsonObject;
   if?: string | boolean;
 }
+
+/**
+ * TaskSpecV1beta3
+ * @public
+ */
 export interface TaskSpecV1beta3 {
   apiVersion: 'scaffolder.backstage.io/v1beta3';
   baseUrl?: string;
@@ -72,16 +98,32 @@ export interface TaskSpecV1beta3 {
   output: { [name: string]: JsonValue };
 }
 
+/**
+ * TaskSpec
+ * @public
+ */
 export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3;
 
+/**
+ * TaskSecrets
+ * @public
+ */
 export type TaskSecrets = {
   token: string | undefined;
 };
 
+/**
+ * DispatchResult
+ * @public
+ */
 export type DispatchResult = {
   taskId: string;
 };
 
+/**
+ * Task
+ * @public
+ */
 export interface Task {
   spec: TaskSpec;
   secrets?: TaskSecrets;
@@ -91,9 +133,16 @@ export interface Task {
   getWorkspaceName(): Promise<string>;
 }
 
+/**
+ * TaskBroker
+ * @public
+ */
 export interface TaskBroker {
   claim(): Promise<Task>;
-  dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise<DispatchResult>;
+  dispatch(
+    spec: TaskSpec,
+    secretTaskSecretss?: TaskSecrets,
+  ): Promise<DispatchResult>;
   vacuumTasks(timeoutS: { timeoutS: number }): Promise<void>;
   observe(
     options: {
@@ -105,8 +154,14 @@ export interface TaskBroker {
       result: { events: DbTaskEventRow[] },
     ) => void,
   ): () => void;
+
+  get(taskId: string): Promise<DbTaskRow>;
 }
 
+/**
+ * TaskStoreEmitOptions
+ * @public
+ */
 export type TaskStoreEmitOptions = {
   taskId: string;
   body: JsonObject;
@@ -117,6 +172,10 @@ export type TaskStoreGetEventsOptions = {
   after?: number | undefined;
 };
 
+/**
+ * TaskStore
+ * @public
+ */
 export interface TaskStore {
   createTask(
     task: TaskSpec,
@@ -142,6 +201,10 @@ export interface TaskStore {
 }
 
 export type WorkflowResponse = { output: { [key: string]: JsonValue } };
+/*
+ * WorkflowRunner
+ * @public
+ */
 export interface WorkflowRunner {
   execute(task: Task): Promise<WorkflowResponse>;
 }

plugins/scaffolder-backend/src/service/router.ts

@@ -25,7 +25,13 @@ import {
   StorageTaskBroker,
   TaskWorker,
 } from '../scaffolder/tasks';
-import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry';
+import {
+  TemplateActionRegistry,
+  TemplateAction,
+  createBuiltinActions,
+  TaskBroker,
+  TaskSpec,
+} from '../scaffolder';
 import { getEntityBaseUrl, getWorkingDirectory } from './helpers';
 import {
   ContainerRunner,
@@ -38,11 +44,9 @@ import { TemplateEntityV1beta2, Entity } from '@backstage/catalog-model';
 import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common';
 
 import { ScmIntegrations } from '@backstage/integration';
-import { TemplateAction } from '../scaffolder/actions';
-import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions';
+
 import { LegacyWorkflowRunner } from '../scaffolder/tasks/LegacyWorkflowRunner';
 import { DefaultWorkflowRunner } from '../scaffolder/tasks/DefaultWorkflowRunner';
-import { TaskSpec } from '../scaffolder/tasks/types';
 
 export interface RouterOptions {
   logger: Logger;
@@ -53,6 +57,7 @@ export interface RouterOptions {
   actions?: TemplateAction<any>[];
   taskWorkers?: number;
   containerRunner: ContainerRunner;
+  taskBroker?: TaskBroker;
 }
 
 function isSupportedTemplate(
@@ -89,7 +94,8 @@ export async function createRouter(
   const databaseTaskStore = await DatabaseTaskStore.create(
     await database.getClient(),
   );
-  const taskBroker = new StorageTaskBroker(databaseTaskStore, logger);
+  const taskBroker =
+    options.taskBroker || new StorageTaskBroker(databaseTaskStore, logger);
   const actionRegistry = new TemplateActionRegistry();
   const legacyWorkflowRunner = new LegacyWorkflowRunner({
     logger,