Merge pull request #20592 from acierto/shutdown-tasks-on-restart

Make it possible to shutdown unfinished scaffolder ongoing tasks after restart

.changeset/pink-eyes-reflect.md

@@ -0,0 +1,10 @@
+---
+'@backstage/plugin-scaffolder-backend': minor
+---
+
+Made shut down stale tasks configurable.
+
+There are two properties exposed:
+
+- `scaffolder.processingInterval` - sets the processing interval for staled tasks.
+- `scaffolder.taskTimeoutJanitorFrequency` - sets the task's heartbeat timeout, when to consider a task to be staled.

plugins/scaffolder-backend/api-report.md

@@ -804,11 +804,11 @@ export class DatabaseTaskStore implements TaskStore {
     >,
   ): Promise<void>;
   // (undocumented)
-  claimTask(): Promise<SerializedTask | undefined>;
+  claimTask(): Promise<SerializedTask_2 | undefined>;
   // (undocumented)
   completeTask(options: {
     taskId: string;
-    status: TaskStatus;
+    status: TaskStatus_2;
     eventBody: JsonObject;
   }): Promise<void>;
   // (undocumented)
@@ -826,16 +826,16 @@ export class DatabaseTaskStore implements TaskStore {
     >,
   ): Promise<void>;
   // (undocumented)
-  getTask(taskId: string): Promise<SerializedTask>;
+  getTask(taskId: string): Promise<SerializedTask_2>;
   // (undocumented)
   heartbeatTask(taskId: string): Promise<void>;
   // (undocumented)
   list(options: { createdBy?: string }): Promise<{
-    tasks: SerializedTask[];
+    tasks: SerializedTask_2[];
   }>;
   // (undocumented)
   listEvents(options: TaskStoreListEventsOptions): Promise<{
-    events: SerializedTaskEvent[];
+    events: SerializedTaskEvent_2[];
   }>;
   // (undocumented)
   listStaleTasks(options: { timeoutS: number }): Promise<{
@@ -876,9 +876,9 @@ export interface RouterOptions {
   // (undocumented)
   actions?: TemplateAction_2<any, any>[];
   // (undocumented)
-  additionalTemplateFilters?: Record<string, TemplateFilter>;
+  additionalTemplateFilters?: Record<string, TemplateFilter_2>;
   // (undocumented)
-  additionalTemplateGlobals?: Record<string, TemplateGlobal>;
+  additionalTemplateGlobals?: Record<string, TemplateGlobal_2>;
   // (undocumented)
   catalogClient: CatalogApi;
   concurrentTasksLimit?: number;
@@ -901,7 +901,7 @@ export interface RouterOptions {
   // (undocumented)
   scheduler?: PluginTaskScheduler;
   // (undocumented)
-  taskBroker?: TaskBroker;
+  taskBroker?: TaskBroker_2;
   // @deprecated (undocumented)
   taskWorkers?: number;
 }

plugins/scaffolder-backend/config.d.ts

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+import { HumanDuration } from '@backstage/types';
+
 export interface Config {
   /** Configuration options for the scaffolder plugin */
   scaffolder?: {
@@ -37,5 +39,21 @@ export interface Config {
      * Set to 0 to disable task workers altogether.
      */
     concurrentTasksLimit?: number;
+
+    /**
+     * Makes sure to auto-expire and clean up things that time out or for other reasons should not be left lingering.
+     *
+     * By default, the frequency is every 5 minutes.
+     */
+    taskTimeoutJanitorFrequency?: HumanDuration;
+
+    /**
+     * Sets the task's heartbeat timeout, when to consider a task to be staled.
+     *
+     * Once task is considered to be staled, the scheduler will shut it down on the next cycle.
+     *
+     * Default value is 24 hours.
+     */
+    taskTimeout?: HumanDuration;
   };
 }

plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts

@@ -23,10 +23,6 @@ import { ConflictError, NotFoundError } from '@backstage/errors';
 import { Knex } from 'knex';
 import { v4 as uuid } from 'uuid';
 import {
-  SerializedTaskEvent,
-  SerializedTask,
-  TaskStatus,
-  TaskEventType,
   TaskStore,
   TaskStoreEmitOptions,
   TaskStoreListEventsOptions,
@@ -34,6 +30,12 @@ import {
   TaskStoreCreateTaskResult,
   TaskStoreShutDownTaskOptions,
 } from './types';
+import {
+  SerializedTaskEvent,
+  SerializedTask,
+  TaskStatus,
+  TaskEventType,
+} from '@backstage/plugin-scaffolder-node';
 import { DateTime } from 'luxon';
 
 const migrationsDir = resolvePackagePath(

plugins/scaffolder-backend/src/service/helpers.ts

@@ -107,7 +107,3 @@ export async function findTemplate(options: {
 
   return template as TemplateEntityV1beta3;
 }
-
-export type TemplateTransform = (
-  template: TemplateEntityV1beta3,
-) => TemplateEntityV1beta3;

plugins/scaffolder-backend/src/service/router.test.ts

@@ -38,7 +38,8 @@ import {
   stringifyEntityRef,
   UserEntity,
 } from '@backstage/catalog-model';
-import { createRouter, DatabaseTaskStore, TaskBroker } from '../index';
+import { createRouter, DatabaseTaskStore } from '../index';
+import { TaskBroker } from '@backstage/plugin-scaffolder-node';
 import { StorageTaskBroker } from '../scaffolder/tasks/StorageTaskBroker';
 import {
   IdentityApiGetIdentityRequest,

plugins/scaffolder-backend/src/service/router.ts

@@ -24,10 +24,10 @@ import {
   stringifyEntityRef,
   UserEntity,
 } from '@backstage/catalog-model';
-import { Config } from '@backstage/config';
+import { Config, readDurationFromConfig } from '@backstage/config';
 import { InputError, NotFoundError, stringifyError } from '@backstage/errors';
 import { ScmIntegrations } from '@backstage/integration';
-import { JsonObject, JsonValue } from '@backstage/types';
+import { HumanDuration, JsonObject, JsonValue } from '@backstage/types';
 import {
   TaskSpec,
   TemplateEntityV1beta3,
@@ -48,11 +48,14 @@ import Router from 'express-promise-router';
 import { validate } from 'jsonschema';
 import { Logger } from 'winston';
 import { z } from 'zod';
-import { TemplateFilter, TemplateGlobal } from '../lib';
+import {
+  TaskBroker,
+  TemplateFilter,
+  TemplateGlobal,
+} from '@backstage/plugin-scaffolder-node';
 import {
   createBuiltinActions,
   DatabaseTaskStore,
-  TaskBroker,
   TaskWorker,
   TemplateActionRegistry,
 } from '../scaffolder';
@@ -74,6 +77,7 @@ import {
   PermissionRule,
 } from '@backstage/plugin-permission-node';
 import { scaffolderActionRules, scaffolderTemplateRules } from './rules';
+import { Duration } from 'luxon';
 
 /**
  *
@@ -212,6 +216,17 @@ function buildDefaultIdentityClient(options: RouterOptions): IdentityApi {
   };
 }
 
+const readDuration = (
+  config: Config,
+  key: string,
+  defaultValue: HumanDuration,
+) => {
+  if (config.has(key)) {
+    return readDurationFromConfig(config, { key });
+  }
+  return defaultValue;
+};
+
 /**
  * A method to create a router for the scaffolder backend plugin.
  * @public
@@ -256,11 +271,21 @@ export async function createRouter(
     if (scheduler && databaseTaskStore.listStaleTasks) {
       await scheduler.scheduleTask({
         id: 'close_stale_tasks',
-        frequency: { cron: '*/5 * * * *' }, // every 5 minutes, also supports Duration
+        frequency: readDuration(
+          config,
+          'scaffolder.taskTimeoutJanitorFrequency',
+          {
+            minutes: 5,
+          },
+        ),
         timeout: { minutes: 15 },
         fn: async () => {
           const { tasks } = await databaseTaskStore.listStaleTasks({
-            timeoutS: 86400,
+            timeoutS: Duration.fromObject(
+              readDuration(config, 'scaffolder.taskTimeout', {
+                hours: 24,
+              }),
+            ).as('seconds'),
           });
 
           for (const task of tasks) {