diff --git a/.changeset/tidy-oranges-allow.md b/.changeset/tidy-oranges-allow.md new file mode 100644 index 0000000000..78d5393cec --- /dev/null +++ b/.changeset/tidy-oranges-allow.md @@ -0,0 +1,12 @@ +--- +'@backstage/plugin-scaffolder-backend': minor +--- + +Added experimental flag for scaffolder to wait for running tasks to complete on shutdown + +Enabling the `EXPERIMENTAL_gracefulShutdown` flag in the scaffolder config will make the +scaffolder block the shutdown process until all running tasks have completed. This is useful +when there is a need to ensure that all tasks have completed before the scaffolder is shut down. + +Please note, that the `TaskWorker` `stop` method is now asynchronous and awaited for the +tasks to complete when the experimental flag is enabled. diff --git a/plugins/scaffolder-backend/config.d.ts b/plugins/scaffolder-backend/config.d.ts index 90040d68f1..a391f97264 100644 --- a/plugins/scaffolder-backend/config.d.ts +++ b/plugins/scaffolder-backend/config.d.ts @@ -41,6 +41,11 @@ export interface Config { */ concurrentTasksLimit?: number; + /** + * Tries to wait for tasks to finish during SIGTERM before shutting down the TaskWorker. + */ + EXPERIMENTAL_gracefulShutdown?: boolean; + /** * Sets the tasks recoverability on system start up. * diff --git a/plugins/scaffolder-backend/report.api.md b/plugins/scaffolder-backend/report.api.md index f21bcd76f7..9d39f52d85 100644 --- a/plugins/scaffolder-backend/report.api.md +++ b/plugins/scaffolder-backend/report.api.md @@ -414,6 +414,7 @@ export type CreateWorkerOptions = { concurrentTasksLimit?: number; additionalTemplateGlobals?: Record; permissions?: PermissionEvaluator; + gracefulShutdown?: boolean; }; // @public @@ -841,7 +842,7 @@ export class TaskWorker { // (undocumented) start(): void; // (undocumented) - stop(): void; + stop(): Promise; } // @public @deprecated (undocumented) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 6c77ef3413..55af873083 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -44,6 +44,7 @@ export type TaskWorkerOptions = { permissions?: PermissionEvaluator; logger?: Logger; auditor?: AuditorService; + gracefulShutdown?: boolean; }; /** @@ -74,6 +75,7 @@ export type CreateWorkerOptions = { concurrentTasksLimit?: number; additionalTemplateGlobals?: Record; permissions?: PermissionEvaluator; + gracefulShutdown?: boolean; }; /** @@ -108,6 +110,7 @@ export class TaskWorker { concurrentTasksLimit = 10, // from 1 to Infinity additionalTemplateGlobals, permissions, + gracefulShutdown, } = options; const workflowRunner = new NunjucksWorkflowRunner({ @@ -127,6 +130,7 @@ export class TaskWorker { concurrentTasksLimit, permissions, auditor, + gracefulShutdown, }); } @@ -156,8 +160,13 @@ export class TaskWorker { })(); } - stop() { + async stop() { this.stopWorkers = true; + if (this.options?.gracefulShutdown) { + while (this.taskQueue.size > 0) { + await new Promise(resolve => setTimeout(resolve, 1000)); + } + } } protected onReadyToClaimTask(): Promise { diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index a12e43615e..c39ee29a1e 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -366,6 +366,10 @@ export async function createRouter( const workers: TaskWorker[] = []; if (concurrentTasksLimit !== 0) { + const gracefulShutdown = config.getOptionalBoolean( + 'scaffolder.EXPERIMENTAL_gracefulShutdown', + ); + for (let i = 0; i < (taskWorkers || 1); i++) { const worker = await TaskWorker.create({ taskBroker, @@ -378,6 +382,7 @@ export async function createRouter( additionalTemplateGlobals, concurrentTasksLimit, permissions, + gracefulShutdown, }); workers.push(worker); } @@ -399,8 +404,8 @@ export async function createRouter( const launchWorkers = () => workers.forEach(worker => worker.start()); - const shutdownWorkers = () => { - workers.forEach(worker => worker.stop()); + const shutdownWorkers = async () => { + await Promise.allSettled(workers.map(worker => worker.stop())); }; if (options.lifecycle) {