Merge pull request #28705 from drodil/hodor

feat(scaffolder): experimental flag to wait for task completions

.changeset/tidy-oranges-allow.md

@@ -0,0 +1,12 @@
+---
+'@backstage/plugin-scaffolder-backend': minor
+---
+
+Added experimental flag for scaffolder to wait for running tasks to complete on shutdown
+
+Enabling the `EXPERIMENTAL_gracefulShutdown` flag in the scaffolder config will make the
+scaffolder block the shutdown process until all running tasks have completed. This is useful
+when there is a need to ensure that all tasks have completed before the scaffolder is shut down.
+
+Please note, that the `TaskWorker` `stop` method is now asynchronous and awaited for the
+tasks to complete when the experimental flag is enabled.

plugins/scaffolder-backend/config.d.ts

@@ -41,6 +41,11 @@ export interface Config {
      */
     concurrentTasksLimit?: number;
 
+    /**
+     * Tries to wait for tasks to finish during SIGTERM before shutting down the TaskWorker.
+     */
+    EXPERIMENTAL_gracefulShutdown?: boolean;
+
     /**
      * Sets the tasks recoverability on system start up.
      *

plugins/scaffolder-backend/report.api.md

@@ -414,6 +414,7 @@ export type CreateWorkerOptions = {
   concurrentTasksLimit?: number;
   additionalTemplateGlobals?: Record<string, TemplateGlobal_2>;
   permissions?: PermissionEvaluator;
+  gracefulShutdown?: boolean;
 };
 
 // @public
@@ -841,7 +842,7 @@ export class TaskWorker {
   // (undocumented)
   start(): void;
   // (undocumented)
-  stop(): void;
+  stop(): Promise<void>;
 }
 
 // @public @deprecated (undocumented)

plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts

@@ -29,6 +29,7 @@ import { Logger } from 'winston';
 import { TemplateActionRegistry } from '../actions';
 import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner';
 import { WorkflowRunner } from './types';
+import { setTimeout } from 'timers/promises';
 
 /**
  * TaskWorkerOptions
@@ -44,6 +45,7 @@ export type TaskWorkerOptions = {
   permissions?: PermissionEvaluator;
   logger?: Logger;
   auditor?: AuditorService;
+  gracefulShutdown?: boolean;
 };
 
 /**
@@ -74,6 +76,7 @@ export type CreateWorkerOptions = {
   concurrentTasksLimit?: number;
   additionalTemplateGlobals?: Record<string, TemplateGlobal>;
   permissions?: PermissionEvaluator;
+  gracefulShutdown?: boolean;
 };
 
 /**
@@ -108,6 +111,7 @@ export class TaskWorker {
       concurrentTasksLimit = 10, // from 1 to Infinity
       additionalTemplateGlobals,
       permissions,
+      gracefulShutdown,
     } = options;
 
     const workflowRunner = new NunjucksWorkflowRunner({
@@ -127,6 +131,7 @@ export class TaskWorker {
       concurrentTasksLimit,
       permissions,
       auditor,
+      gracefulShutdown,
     });
   }
 
@@ -141,7 +146,7 @@ export class TaskWorker {
   start() {
     (async () => {
       while (!this.stopWorkers) {
-        await new Promise(resolve => setTimeout(resolve, 10000));
+        await setTimeout(10000);
         await this.recoverTasks();
       }
     })();
@@ -156,8 +161,13 @@ export class TaskWorker {
     })();
   }
 
-  stop() {
+  async stop() {
     this.stopWorkers = true;
+    if (this.options?.gracefulShutdown) {
+      while (this.taskQueue.size > 0) {
+        await setTimeout(1000);
+      }
+    }
   }
 
   protected onReadyToClaimTask(): Promise<void> {

plugins/scaffolder-backend/src/service/router.ts

@@ -366,6 +366,10 @@ export async function createRouter(
 
   const workers: TaskWorker[] = [];
   if (concurrentTasksLimit !== 0) {
+    const gracefulShutdown = config.getOptionalBoolean(
+      'scaffolder.EXPERIMENTAL_gracefulShutdown',
+    );
+
     for (let i = 0; i < (taskWorkers || 1); i++) {
       const worker = await TaskWorker.create({
         taskBroker,
@@ -378,6 +382,7 @@ export async function createRouter(
         additionalTemplateGlobals,
         concurrentTasksLimit,
         permissions,
+        gracefulShutdown,
       });
       workers.push(worker);
     }
@@ -399,8 +404,8 @@ export async function createRouter(
 
   const launchWorkers = () => workers.forEach(worker => worker.start());
 
-  const shutdownWorkers = () => {
-    workers.forEach(worker => worker.stop());
+  const shutdownWorkers = async () => {
+    await Promise.allSettled(workers.map(worker => worker.stop()));
   };
 
   if (options.lifecycle) {