feat(scaffolder): experimental flag to wait for task completions

Enabling the `EXPERIMENTAL_gracefulShutdown` flag in the scaffolder config will make the
scaffolder block the shutdown process until all running tasks have completed. This is useful
when there is a need to ensure that all tasks have completed before the scaffolder is shut down.

Signed-off-by: Hellgren Heikki <heikki.hellgren@op.fi>

.changeset/tidy-oranges-allow.md

@@ -0,0 +1,12 @@
+---
+'@backstage/plugin-scaffolder-backend': minor
+---
+
+Added experimental flag for scaffolder to wait for running tasks to complete on shutdown
+
+Enabling the `EXPERIMENTAL_gracefulShutdown` flag in the scaffolder config will make the
+scaffolder block the shutdown process until all running tasks have completed. This is useful
+when there is a need to ensure that all tasks have completed before the scaffolder is shut down.
+
+Please note, that the `TaskWorker` `stop` method is now asynchronous and awaited for the
+tasks to complete when the experimental flag is enabled.

plugins/scaffolder-backend/config.d.ts

@@ -41,6 +41,11 @@ export interface Config {
      */
     concurrentTasksLimit?: number;
 
+    /**
+     * Tries to wait for tasks to finish during SIGTERM before shutting down the TaskWorker.
+     */
+    EXPERIMENTAL_gracefulShutdown?: boolean;
+
     /**
      * Sets the tasks recoverability on system start up.
      *

plugins/scaffolder-backend/report.api.md

@@ -414,6 +414,7 @@ export type CreateWorkerOptions = {
   concurrentTasksLimit?: number;
   additionalTemplateGlobals?: Record<string, TemplateGlobal_2>;
   permissions?: PermissionEvaluator;
+  gracefulShutdown?: boolean;
 };
 
 // @public
@@ -841,7 +842,7 @@ export class TaskWorker {
   // (undocumented)
   start(): void;
   // (undocumented)
-  stop(): void;
+  stop(): Promise<void>;
 }
 
 // @public @deprecated (undocumented)

plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts

@@ -44,6 +44,7 @@ export type TaskWorkerOptions = {
   permissions?: PermissionEvaluator;
   logger?: Logger;
   auditor?: AuditorService;
+  gracefulShutdown?: boolean;
 };
 
 /**
@@ -74,6 +75,7 @@ export type CreateWorkerOptions = {
   concurrentTasksLimit?: number;
   additionalTemplateGlobals?: Record<string, TemplateGlobal>;
   permissions?: PermissionEvaluator;
+  gracefulShutdown?: boolean;
 };
 
 /**
@@ -108,6 +110,7 @@ export class TaskWorker {
       concurrentTasksLimit = 10, // from 1 to Infinity
       additionalTemplateGlobals,
       permissions,
+      gracefulShutdown,
     } = options;
 
     const workflowRunner = new NunjucksWorkflowRunner({
@@ -127,6 +130,7 @@ export class TaskWorker {
       concurrentTasksLimit,
       permissions,
       auditor,
+      gracefulShutdown,
     });
   }
 
@@ -156,8 +160,13 @@ export class TaskWorker {
     })();
   }
 
-  stop() {
+  async stop() {
     this.stopWorkers = true;
+    if (this.options?.gracefulShutdown) {
+      while (this.taskQueue.size > 0) {
+        await new Promise(resolve => setTimeout(resolve, 1000));
+      }
+    }
   }
 
   protected onReadyToClaimTask(): Promise<void> {

plugins/scaffolder-backend/src/service/router.ts

@@ -366,6 +366,10 @@ export async function createRouter(
 
   const workers: TaskWorker[] = [];
   if (concurrentTasksLimit !== 0) {
+    const gracefulShutdown = config.getOptionalBoolean(
+      'scaffolder.EXPERIMENTAL_gracefulShutdown',
+    );
+
     for (let i = 0; i < (taskWorkers || 1); i++) {
       const worker = await TaskWorker.create({
         taskBroker,
@@ -378,6 +382,7 @@ export async function createRouter(
         additionalTemplateGlobals,
         concurrentTasksLimit,
         permissions,
+        gracefulShutdown,
       });
       workers.push(worker);
     }
@@ -399,8 +404,8 @@ export async function createRouter(
 
   const launchWorkers = () => workers.forEach(worker => worker.start());
 
-  const shutdownWorkers = () => {
-    workers.forEach(worker => worker.stop());
+  const shutdownWorkers = async () => {
+    await Promise.allSettled(workers.map(worker => worker.stop()));
   };
 
   if (options.lifecycle) {