From 7d0d43c51a40143c1482b87b2fafbfbfd700f9ae Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Mon, 2 Dec 2024 17:07:51 +0100 Subject: [PATCH] fix: drain request after shutdown Signed-off-by: Camila Belo --- .../src/wiring/BackendInitializer.ts | 33 ++++++++++++++----- .../rootHealthServiceFactory.test.ts | 18 ++++++++++ .../rootHealth/rootHealthServiceFactory.ts | 3 +- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.ts b/packages/backend-app-api/src/wiring/BackendInitializer.ts index 3e7b46fc9e..101c9abeb7 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.ts @@ -65,9 +65,18 @@ const instanceRegistry = new (class InstanceRegistry { if (!this.#registered) { this.#registered = true; - process.addListener('SIGTERM', this.#exitHandler); - process.addListener('SIGINT', this.#exitHandler); - process.addListener('beforeExit', this.#exitHandler); + process.addListener('SIGTERM', () => { + this.#exitHandler(); + }); + process.addListener('SIGINT', () => { + this.#exitHandler(); + }); + process.addListener('SIGQUIT', () => { + process.exit(0); + }); + process.addListener('beforeExit', () => { + this.#exitHandler(); + }); } this.#instances.add(instance); @@ -79,12 +88,18 @@ const instanceRegistry = new (class InstanceRegistry { #exitHandler = async () => { try { - const results = await Promise.allSettled( - Array.from(this.#instances).map(b => b.stop()), - ); - const errors = results.flatMap(r => - r.status === 'rejected' ? [r.reason] : [], - ); + // This signals to the healthcheck service that the process is shutting down + process.exitCode = 0; + + const results = await Promise.race([ + // Give the backend 30 seconds to shut down, then force exit + new Promise(resolve => setTimeout(resolve, 30000)), + Promise.allSettled(Array.from(this.#instances).map(b => b.stop())), + ]); + + const errors = Array.isArray(results) + ? results.flatMap(r => (r.status === 'rejected' ? [r.reason] : [])) + : []; if (errors.length > 0) { for (const error of errors) { diff --git a/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.test.ts b/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.test.ts index 0eee5c18c6..c84c225fac 100644 --- a/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.test.ts +++ b/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.test.ts @@ -51,6 +51,24 @@ describe('DefaultRootHealthService', () => { }); }); + it(`should return a 503 response if the server is stopping`, async () => { + const service = new DefaultRootHealthService({ + lifecycle: mockServices.rootLifecycle.mock(), + }); + + process.exitCode = 1; + + await expect(service.getReadiness()).resolves.toEqual({ + status: 503, + payload: { + message: 'Backend has not started yet', + status: 'error', + }, + }); + + process.exitCode = undefined; // Reset exitCode for other tests + }); + it(`should return a 500 response if the server has stopped`, async () => { let mockServerStartedFn = () => {}; let mockServerStoppedFn = () => {}; diff --git a/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.ts b/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.ts index c8b4ad5394..b37a6e589f 100644 --- a/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.ts @@ -39,7 +39,8 @@ export class DefaultRootHealthService implements RootHealthService { } async getReadiness(): Promise<{ status: number; payload?: any }> { - if (!this.#isRunning) { + // If the process has been told to exit, or if the backend has not started yet + if (process.exitCode !== undefined || !this.#isRunning) { return { status: 503, payload: { message: 'Backend has not started yet', status: 'error' },