diff --git a/.changeset/green-carpets-smell.md b/.changeset/green-carpets-smell.md new file mode 100644 index 0000000000..0cda68de22 --- /dev/null +++ b/.changeset/green-carpets-smell.md @@ -0,0 +1,7 @@ +--- +'@backstage/backend-plugin-api': patch +--- + +The `RootLifecycleService` now has a new `addBeforeShutdownHook` method, and hooks added through this method will run immediately when a termination event is received. + +The backend will not proceed with the shutdown and run the `Shutdown` hooks until all `BeforeShutdown` hooks have completed. diff --git a/.changeset/large-birds-watch.md b/.changeset/large-birds-watch.md new file mode 100644 index 0000000000..24d63492bc --- /dev/null +++ b/.changeset/large-birds-watch.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-test-utils': patch +--- + +Mock the new `RootLifecycleService.addBeforeShutdownHook` method. diff --git a/.changeset/pretty-kiwis-travel.md b/.changeset/pretty-kiwis-travel.md new file mode 100644 index 0000000000..f3d5cbeb27 --- /dev/null +++ b/.changeset/pretty-kiwis-travel.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-defaults': patch +--- + +Implements the `DefaultRootLifecycleService.addBeforeShutdownHook` method, and updates `DefaultRootHttpRouterService` and `DefaultRootHealthService` to listen to that event to stop accepting traffic and close service connections. diff --git a/.changeset/rude-pears-tie.md b/.changeset/rude-pears-tie.md new file mode 100644 index 0000000000..c0cb970727 --- /dev/null +++ b/.changeset/rude-pears-tie.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-defaults': patch +--- + +Remove use of the `stoppable` library on the `DefaultRootHttpRouterService` as Node's native http server [close](https://nodejs.org/api/http.html#serverclosecallback) method already drains requests. +Also, we pass a new `lifecycleMiddleware` to the `rootHttpRouterServiceFactory` configure function that must be called manually if you don't call `applyDefaults`. diff --git a/.changeset/young-cobras-add.md b/.changeset/young-cobras-add.md new file mode 100644 index 0000000000..f9172d3800 --- /dev/null +++ b/.changeset/young-cobras-add.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-app-api': patch +--- + +As soon as a backend termination signal is received, call before shutting down root lifecycle hooks. diff --git a/docs/backend-system/core-services/root-http-router.md b/docs/backend-system/core-services/root-http-router.md index a2316b957b..9ee8599eb5 100644 --- a/docs/backend-system/core-services/root-http-router.md +++ b/docs/backend-system/core-services/root-http-router.md @@ -42,6 +42,29 @@ createBackendPlugin({ ## Configuring the service +### Via `app-config.yaml` + +The `app-config.yaml` file provides configurable options that can be adjusted to meet your `RootHttpRouterService` specific requirements: + +```yaml +backend: + lifecycle: + # (Optional) The maximum time that paused requests will wait for the service to start, before returning an error (defaults to 5 seconds). + # Supported formats: + # - A string in the format of '1d', '2 seconds' etc. as supported by the `ms` library. + # - A standard ISO formatted duration string, e.g. 'P2DT6H' or 'PT1M'. + # - An object with individual units (in plural) as keys, e.g. `{ days: 2, hours: 6 }`. + startupRequestPauseTimeout: { seconds: 10 } + # (Optional) The minimum time that the HTTP server will delay the shutdown of the backend. During this delay health checks will be set to failing, allowing traffic to drain (defaults to 0 seconds). + # Supported formats: + # - A string in the format of '1d', '2 seconds' etc. as supported by the `ms` library. + # - A standard ISO formatted duration string, e.g. 'P2DT6H' or 'PT1M'. + # - An object with individual units (in plural) as keys, e.g. `{ days: 2, hours: 6 }`. + serverShutdownTimeout: { seconds: 20 } +``` + +### Via Code + There's additional options that you can pass to configure the root HTTP Router service. These options are passed when you call `createBackend`. - `indexPath` - optional path to forward all unmatched requests to. Defaults to `/api/app` which is the `app-backend` plugin responsible for serving the frontend application through the backend. diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.ts b/packages/backend-app-api/src/wiring/BackendInitializer.ts index 440a334d9b..b8c37f7471 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.ts @@ -449,6 +449,11 @@ export class BackendInitializer { // The startup failed, but we may still want to do cleanup so we continue silently } + const rootLifecycleService = await this.#getRootLifecycleImpl(); + + // Root services like the health one need to immediatelly be notified of the shutdown + await rootLifecycleService.beforeShutdown(); + // Get all plugins. const allPlugins = new Set(); for (const feature of this.#registrations) { @@ -468,14 +473,14 @@ export class BackendInitializer { ); // Once all plugin shutdown hooks are done, run root shutdown hooks. - const lifecycleService = await this.#getRootLifecycleImpl(); - await lifecycleService.shutdown(); + await rootLifecycleService.shutdown(); } // Bit of a hacky way to grab the lifecycle services, potentially find a nicer way to do this async #getRootLifecycleImpl(): Promise< RootLifecycleService & { startup(): Promise; + beforeShutdown(): Promise; shutdown(): Promise; } > { diff --git a/packages/backend-app-api/src/wiring/createSpecializedBackend.test.ts b/packages/backend-app-api/src/wiring/createSpecializedBackend.test.ts index 1fd7fcbbec..32c1fbcd0e 100644 --- a/packages/backend-app-api/src/wiring/createSpecializedBackend.test.ts +++ b/packages/backend-app-api/src/wiring/createSpecializedBackend.test.ts @@ -36,6 +36,7 @@ describe('createSpecializedBackend', () => { deps: {}, factory: async () => ({ addStartupHook: () => {}, + addBeforeShutdownHook: () => {}, addShutdownHook: () => {}, }), }), @@ -44,6 +45,7 @@ describe('createSpecializedBackend', () => { deps: {}, factory: async () => ({ addStartupHook: () => {}, + addBeforeShutdownHook: () => {}, addShutdownHook: () => {}, }), }), diff --git a/packages/backend-defaults/config.d.ts b/packages/backend-defaults/config.d.ts index 0f7f9906f8..e2a748f85d 100644 --- a/packages/backend-defaults/config.d.ts +++ b/packages/backend-defaults/config.d.ts @@ -28,6 +28,29 @@ export interface Config { */ baseUrl: string; + lifecycle?: { + /** + * The maximum time that paused requests will wait for the service to start, before returning an error. + * Defaults to 5 seconds + * Supported formats: + * - A string in the format of '1d', '2 seconds' etc. as supported by the `ms` + * library. + * - A standard ISO formatted duration string, e.g. 'P2DT6H' or 'PT1M'. + * - An object with individual units (in plural) as keys, e.g. `{ days: 2, hours: 6 }`. + */ + startupRequestPauseTimeout?: string | HumanDuration; + /** + * The minimum time that the HTTP server will delay the shutdown of the backend. During this delay health checks will be set to failing, allowing traffic to drain. + * Defaults to 0 seconds. + * Supported formats: + * - A string in the format of '1d', '2 seconds' etc. as supported by the `ms` + * library. + * - A standard ISO formatted duration string, e.g. 'P2DT6H' or 'PT1M'. + * - An object with individual units (in plural) as keys, e.g. `{ days: 2, hours: 6 }`. + */ + serverShutdownDelay?: string | HumanDuration; + }; + /** Address that the backend should listen to. */ listen?: | string diff --git a/packages/backend-defaults/package.json b/packages/backend-defaults/package.json index d8240c429c..88d3ca79b1 100644 --- a/packages/backend-defaults/package.json +++ b/packages/backend-defaults/package.json @@ -175,7 +175,6 @@ "pg-format": "^1.0.4", "raw-body": "^2.4.1", "selfsigned": "^2.0.0", - "stoppable": "^1.1.0", "tar": "^6.1.12", "triple-beam": "^1.4.1", "uuid": "^11.0.0", diff --git a/packages/backend-defaults/report-rootHttpRouter.api.md b/packages/backend-defaults/report-rootHttpRouter.api.md index 41830ebf17..4489fed107 100644 --- a/packages/backend-defaults/report-rootHttpRouter.api.md +++ b/packages/backend-defaults/report-rootHttpRouter.api.md @@ -134,6 +134,8 @@ export interface RootHttpRouterConfigureContext { // (undocumented) lifecycle: LifecycleService; // (undocumented) + lifecycleMiddleware: RequestHandler; + // (undocumented) logger: LoggerService; // (undocumented) middleware: MiddlewareFactory; diff --git a/packages/backend-defaults/src/CreateBackend.test.ts b/packages/backend-defaults/src/CreateBackend.test.ts index 55fdbe5a6a..57977e9647 100644 --- a/packages/backend-defaults/src/CreateBackend.test.ts +++ b/packages/backend-defaults/src/CreateBackend.test.ts @@ -47,6 +47,7 @@ describe('createBackend', () => { deps: {}, factory: async () => ({ addStartupHook: () => {}, + addBeforeShutdownHook: () => {}, addShutdownHook: () => {}, }), }), @@ -57,6 +58,7 @@ describe('createBackend', () => { deps: {}, factory: async () => ({ addStartupHook: () => {}, + addBeforeShutdownHook: () => {}, addShutdownHook: () => {}, }), }), diff --git a/packages/backend-defaults/src/entrypoints/httpRouter/httpRouterServiceFactory.ts b/packages/backend-defaults/src/entrypoints/httpRouter/httpRouterServiceFactory.ts index 0557cb636d..a448827e16 100644 --- a/packages/backend-defaults/src/entrypoints/httpRouter/httpRouterServiceFactory.ts +++ b/packages/backend-defaults/src/entrypoints/httpRouter/httpRouterServiceFactory.ts @@ -22,7 +22,6 @@ import { HttpRouterServiceAuthPolicy, } from '@backstage/backend-plugin-api'; import { - createLifecycleMiddleware, createCookieAuthRefreshMiddleware, createCredentialsBarrier, createAuthIntegrationRouter, @@ -43,12 +42,11 @@ export const httpRouterServiceFactory = createServiceFactory({ deps: { plugin: coreServices.pluginMetadata, config: coreServices.rootConfig, - lifecycle: coreServices.lifecycle, rootHttpRouter: coreServices.rootHttpRouter, auth: coreServices.auth, httpAuth: coreServices.httpAuth, }, - async factory({ auth, httpAuth, config, plugin, rootHttpRouter, lifecycle }) { + async factory({ auth, httpAuth, config, plugin, rootHttpRouter }) { const router = PromiseRouter(); rootHttpRouter.use(`/api/${plugin.getId()}`, router); @@ -59,7 +57,6 @@ export const httpRouterServiceFactory = createServiceFactory({ }); router.use(createAuthIntegrationRouter({ auth })); - router.use(createLifecycleMiddleware({ lifecycle })); router.use(credentialsBarrier.middleware); router.use(createCookieAuthRefreshMiddleware({ auth, httpAuth })); diff --git a/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.test.ts b/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.test.ts index 0eee5c18c6..ca9085273c 100644 --- a/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.test.ts +++ b/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.test.ts @@ -53,11 +53,11 @@ describe('DefaultRootHealthService', () => { it(`should return a 500 response if the server has stopped`, async () => { let mockServerStartedFn = () => {}; - let mockServerStoppedFn = () => {}; + let mockServerBeforeStoppedFn = () => {}; const lifecycle = mockServices.rootLifecycle.mock({ addStartupHook: jest.fn(fn => (mockServerStartedFn = fn)), - addShutdownHook: jest.fn(fn => (mockServerStoppedFn = fn)), + addBeforeShutdownHook: jest.fn(fn => (mockServerBeforeStoppedFn = fn)), }); const service = new DefaultRootHealthService({ @@ -65,11 +65,11 @@ describe('DefaultRootHealthService', () => { }); mockServerStartedFn(); - mockServerStoppedFn(); + mockServerBeforeStoppedFn(); await expect(service.getReadiness()).resolves.toEqual({ status: 503, payload: { - message: 'Backend has not started yet', + message: 'Backend is shuttting down', status: 'error', }, }); diff --git a/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.ts b/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.ts index c8b4ad5394..b201079b4f 100644 --- a/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHealth/rootHealthServiceFactory.ts @@ -23,14 +23,14 @@ import { /** @internal */ export class DefaultRootHealthService implements RootHealthService { - #isRunning = false; + #state: 'init' | 'up' | 'down' = 'init'; constructor(readonly options: { lifecycle: RootLifecycleService }) { options.lifecycle.addStartupHook(() => { - this.#isRunning = true; + this.#state = 'up'; }); - options.lifecycle.addShutdownHook(() => { - this.#isRunning = false; + options.lifecycle.addBeforeShutdownHook(() => { + this.#state = 'down'; }); } @@ -39,12 +39,18 @@ export class DefaultRootHealthService implements RootHealthService { } async getReadiness(): Promise<{ status: number; payload?: any }> { - if (!this.#isRunning) { + if (this.#state === 'init') { return { status: 503, payload: { message: 'Backend has not started yet', status: 'error' }, }; } + if (this.#state === 'down') { + return { + status: 503, + payload: { message: 'Backend is shuttting down', status: 'error' }, + }; + } return { status: 200, payload: { status: 'ok' } }; } diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/createLifecycleMiddleware.test.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/createLifecycleMiddleware.test.ts new file mode 100644 index 0000000000..76af28cc6e --- /dev/null +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/createLifecycleMiddleware.test.ts @@ -0,0 +1,108 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createLifecycleMiddleware } from './createLifecycleMiddleware'; +import { BackendLifecycleImpl } from '../rootLifecycle/rootLifecycleServiceFactory'; +import { mockServices } from '@backstage/backend-test-utils'; +import { ServiceUnavailableError } from '@backstage/errors'; + +describe('createLifecycleMiddleware', () => { + it('should pause requests when plugin is not ready', async () => { + const lifecycle = new BackendLifecycleImpl(mockServices.rootLogger()); + + const middleware = createLifecycleMiddleware({ lifecycle }); + + const next = jest.fn(); + middleware({} as any, {} as any, next); + expect(next).not.toHaveBeenCalled(); + await lifecycle.startup(); + + // pending call + expect(next).toHaveBeenCalledWith(); + + // new call + const next2 = jest.fn(); + middleware({} as any, {} as any, next2); + expect(next2).toHaveBeenCalledWith(); + }); + + it('should throw ServiceUnavailableError after shutdown', async () => { + const lifecycle = new BackendLifecycleImpl(mockServices.rootLogger()); + const middleware = createLifecycleMiddleware({ lifecycle }); + + const next = jest.fn(); + middleware({} as any, {} as any, next); + expect(next).not.toHaveBeenCalled(); + await lifecycle.shutdown(); + + // pending call + expect(next).toHaveBeenCalledWith( + new ServiceUnavailableError('Service is shutting down'), + ); + + // new call + const next2 = jest.fn(); + middleware({} as any, {} as any, next2); + expect(next2).toHaveBeenCalledWith( + new ServiceUnavailableError('Service is shutting down'), + ); + }); + + it('should throw ServiceUnavailableError after timeout', async () => { + const lifecycle = new BackendLifecycleImpl(mockServices.rootLogger()); + const middleware = createLifecycleMiddleware({ + lifecycle, + startupRequestPauseTimeout: { milliseconds: 1 }, + }); + + const next = jest.fn(); + middleware({} as any, {} as any, next); + expect(next).not.toHaveBeenCalled(); + + await new Promise(r => setTimeout(r, 2)); + + expect(next).toHaveBeenCalledWith( + new ServiceUnavailableError('Service has not started up yet'), + ); + }); + + it('should delay service shutdown for the default timeout duration', async () => { + jest.useFakeTimers(); + const defaultTimeout = 30000; + const lifecycle = new BackendLifecycleImpl(mockServices.rootLogger()); + createLifecycleMiddleware({ lifecycle }); + const beforeShutdownPromise = lifecycle.beforeShutdown().then(() => { + jest.useRealTimers(); + }); + jest.advanceTimersByTime(defaultTimeout); + return expect(beforeShutdownPromise).resolves.toBeUndefined(); + }); + + it('should delay service shutdown for the configured timeout duration - time in human duration', async () => { + jest.useFakeTimers(); + const configuredTimeout = 20000; + const lifecycle = new BackendLifecycleImpl(mockServices.rootLogger()); + createLifecycleMiddleware({ + lifecycle, + serverShutdownDelay: { milliseconds: configuredTimeout }, + }); + const beforeShutdownPromise = lifecycle.beforeShutdown().then(() => { + jest.useRealTimers(); + }); + jest.advanceTimersByTime(configuredTimeout); + return expect(beforeShutdownPromise).resolves.toBeUndefined(); + }); +}); diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/createLifecycleMiddleware.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/createLifecycleMiddleware.ts new file mode 100644 index 0000000000..3745f7f48d --- /dev/null +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/createLifecycleMiddleware.ts @@ -0,0 +1,124 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { RootLifecycleService } from '@backstage/backend-plugin-api'; +import { ServiceUnavailableError } from '@backstage/errors'; +import { HumanDuration, durationToMilliseconds } from '@backstage/types'; +import { RequestHandler } from 'express'; + +export const DEFAULT_STARTUP_REQUEST_PAUSE_TIMEOUT = { seconds: 5 }; +export const DEFAULT_SERVER_SHUTDOWN_TIMEOUT = { seconds: 0 }; + +/** + * Options for {@link createLifecycleMiddleware}. + * @public + */ +export interface LifecycleMiddlewareOptions { + lifecycle: RootLifecycleService; + /** + * The maximum time that paused requests will wait for the service to start, before returning an error. + * + * Defaults to 5 seconds. + */ + startupRequestPauseTimeout?: HumanDuration; + /** + * The maximum time that the server will wait for stop accepting traffic, before returning an error. + * + * Defaults to 0 seconds. + */ + serverShutdownDelay?: HumanDuration; +} + +/** + * Creates a middleware that pauses requests until the service has started. + * + * @remarks + * + * Requests that arrive before the service has started will be paused until startup is complete. + * If the service does not start within the provided timeout, the request will be rejected with a + * {@link @backstage/errors#ServiceUnavailableError}. + * + * If the service is shutting down, all requests will be rejected with a + * {@link @backstage/errors#ServiceUnavailableError}. + * + * @public + */ +export function createLifecycleMiddleware( + options: LifecycleMiddlewareOptions, +): RequestHandler { + const { lifecycle, startupRequestPauseTimeout, serverShutdownDelay } = + options; + + let state: 'init' | 'up' | 'down' = 'init'; + const waiting = new Set<{ + next: (err?: Error) => void; + timeout: NodeJS.Timeout; + }>(); + + lifecycle.addStartupHook(async () => { + if (state === 'init') { + state = 'up'; + for (const item of waiting) { + clearTimeout(item.timeout); + item.next(); + } + waiting.clear(); + } + }); + + lifecycle.addBeforeShutdownHook(async () => { + const timeoutMs = durationToMilliseconds( + serverShutdownDelay ?? DEFAULT_SERVER_SHUTDOWN_TIMEOUT, + ); + return await new Promise(resolve => { + setTimeout(resolve, timeoutMs); + }); + }); + + lifecycle.addShutdownHook(async () => { + state = 'down'; + for (const item of waiting) { + clearTimeout(item.timeout); + item.next(new ServiceUnavailableError('Service is shutting down')); + } + waiting.clear(); + }); + + return (_req, _res, next) => { + if (state === 'up') { + next(); + return; + } else if (state === 'down') { + next(new ServiceUnavailableError('Service is shutting down')); + return; + } + + const timeoutMs = durationToMilliseconds( + startupRequestPauseTimeout ?? DEFAULT_STARTUP_REQUEST_PAUSE_TIMEOUT, + ); + + const item = { + next, + timeout: setTimeout(() => { + if (waiting.delete(item)) { + next(new ServiceUnavailableError('Service has not started up yet')); + } + }, timeoutMs), + }; + + waiting.add(item); + }; +} diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/createHttpServer.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/createHttpServer.ts index 5c526ba1e2..9449291628 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/createHttpServer.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/createHttpServer.ts @@ -16,7 +16,6 @@ import * as http from 'http'; import * as https from 'https'; -import stoppableServer from 'stoppable'; import { RequestListener } from 'http'; import { LoggerService } from '@backstage/backend-plugin-api'; import { HttpServerOptions, ExtendedHttpServer } from './types'; @@ -33,13 +32,6 @@ export async function createHttpServer( deps: { logger: LoggerService }, ): Promise { const server = await createServer(listener, options, deps); - - const stopper = stoppableServer(server, 0); - // The stopper here is actually the server itself, so if we try - // to call stopper.stop() down in the stop implementation, we'll - // be calling ourselves. - const stopServer = stopper.stop.bind(stopper); - return Object.assign(server, { start() { return new Promise((resolve, reject) => { @@ -61,7 +53,7 @@ export async function createHttpServer( stop() { return new Promise((resolve, reject) => { - stopServer((error?: Error) => { + server.close(error => { if (error) { reject(error); } else { diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.test.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.test.ts index f058c87900..fb0d6cdad0 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.test.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.test.ts @@ -21,7 +21,12 @@ import { import { Express } from 'express'; import request from 'supertest'; import { rootHttpRouterServiceFactory } from './rootHttpRouterServiceFactory'; -import { ServiceFactory, coreServices } from '@backstage/backend-plugin-api'; +import { + ServiceFactory, + coreServices, + createServiceFactory, +} from '@backstage/backend-plugin-api'; +import { BackendLifecycleImpl } from '../rootLifecycle/rootLifecycleServiceFactory'; async function createExpressApp(...dependencies: ServiceFactory[]) { let app: Express | undefined = undefined; @@ -198,4 +203,130 @@ describe('rootHttpRouterServiceFactory', () => { 'Invalid header value in at backend.health.headers, must be a non-empty string', ); }); + + it('should wait the server to shutdown', async () => { + jest.useFakeTimers(); + + let app: Express | undefined = undefined; + const lifecycleMock = new BackendLifecycleImpl(mockServices.rootLogger()); + + const tester = ServiceFactoryTester.from( + rootHttpRouterServiceFactory({ + configure(options) { + console.log('configure'); + options.app.use(options.healthRouter); + options.app.use(options.lifecycleMiddleware); + options.app.get('/test', (_req, res) => { + res.status(200).send({ status: 'ok' }).end(); + }); + options.app.use(options.middleware.error()); + app = options.app; + }, + }), + { + dependencies: [ + mockServices.rootConfig.factory({ + data: { + app: { baseUrl: 'http://localhost' }, + backend: { + baseUrl: 'http://localhost', + listen: { host: '', port: 0 }, + lifecycle: { + serverShutdownDelay: '30s', + }, + }, + }, + }), + createServiceFactory({ + service: coreServices.rootLifecycle, + deps: {}, + factory() { + return lifecycleMock; + }, + }), + ], + }, + ); + + await tester.getSubject(); + + // Trigger creation of the http service, accessing the app instance through the configure callback + const lifecycle = await tester.getService(coreServices.rootLifecycle); + + await (lifecycle as any).startup(); // Trigger startup by calling the private startup method + + await request(app!).get('/test').expect(200, { + status: 'ok', + }); + + await request(app) + .get('/.backstage/health/v1/liveness') + .expect(200, { status: 'ok' }); + + await request(app).get('/.backstage/health/v1/readiness').expect(200, { + status: 'ok', + }); + + const beforeShutdownPromise = (lifecycle as any) + .beforeShutdown() + .then(() => { + return (lifecycle as any).shutdown(); + }); + + // Continue accepting requests + await request(app!).get('/test').expect(200, { + status: 'ok', + }); + + await request(app) + .get('/.backstage/health/v1/liveness') + .expect(200, { status: 'ok' }); + + // Immediately start failing the readiness health check + await request(app).get('/.backstage/health/v1/readiness').expect(503, { + message: 'Backend is shuttting down', + status: 'error', + }); + + jest.advanceTimersByTime(29999); + + // Still accepting requests 1 ms before shutdown + await request(app!).get('/test').expect(200, { + status: 'ok', + }); + + await request(app) + .get('/.backstage/health/v1/liveness') + .expect(200, { status: 'ok' }); + + await request(app).get('/.backstage/health/v1/readiness').expect(503, { + message: 'Backend is shuttting down', + status: 'error', + }); + + jest.advanceTimersByTime(1); + + // No longer accepting requests after shutdown + await request(app!) + .get('/test') + .expect(503, { + error: { + name: 'ServiceUnavailableError', + message: 'Service is shutting down', + }, + request: { method: 'GET', url: '/test' }, + response: { statusCode: 503 }, + }); + + await request(app) + .get('/.backstage/health/v1/liveness') + .expect(200, { status: 'ok' }); + + await request(app).get('/.backstage/health/v1/readiness').expect(503, { + message: 'Backend is shuttting down', + status: 'error', + }); + + return await expect(beforeShutdownPromise).resolves.toBeUndefined(); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.ts index 4e3483d67e..4afdfb1ec2 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.ts @@ -30,6 +30,9 @@ import { } from './http'; import { DefaultRootHttpRouter } from './DefaultRootHttpRouter'; import { createHealthRouter } from './createHealthRouter'; +import { createLifecycleMiddleware } from './createLifecycleMiddleware'; +import { readDurationFromConfig } from '@backstage/config'; +import { HumanDuration } from '@backstage/types'; /** * @public @@ -43,6 +46,7 @@ export interface RootHttpRouterConfigureContext { logger: LoggerService; lifecycle: LifecycleService; healthRouter: RequestHandler; + lifecycleMiddleware: RequestHandler; applyDefaults: () => void; } @@ -90,6 +94,27 @@ const rootHttpRouterServiceFactoryWithOptions = ( const routes = router.handler(); const healthRouter = createHealthRouter({ config, health }); + + let startupRequestPauseTimeout: HumanDuration | undefined; + if (config.has('backend.lifecycle.startupRequestPauseTimeout')) { + startupRequestPauseTimeout = readDurationFromConfig(config, { + key: 'backend.lifecycle.startupRequestPauseTimeout', + }); + } + + let serverShutdownDelay: HumanDuration | undefined; + if (config.has('backend.lifecycle.serverShutdownDelay')) { + serverShutdownDelay = readDurationFromConfig(config, { + key: 'backend.lifecycle.serverShutdownDelay', + }); + } + + const lifecycleMiddleware = createLifecycleMiddleware({ + lifecycle, + startupRequestPauseTimeout, + serverShutdownDelay, + }); + const server = await createHttpServer( app, readHttpServerOptions(config.getOptionalConfig('backend')), @@ -105,6 +130,7 @@ const rootHttpRouterServiceFactoryWithOptions = ( logger, lifecycle, healthRouter, + lifecycleMiddleware, applyDefaults() { if (process.env.NODE_ENV === 'development') { app.set('json spaces', 2); @@ -114,6 +140,7 @@ const rootHttpRouterServiceFactoryWithOptions = ( app.use(middleware.compression()); app.use(middleware.logging()); app.use(healthRouter); + app.use(lifecycleMiddleware); app.use(routes); app.use(middleware.notFound()); app.use(middleware.error()); diff --git a/packages/backend-defaults/src/entrypoints/rootLifecycle/rootLifecycleServiceFactory.ts b/packages/backend-defaults/src/entrypoints/rootLifecycle/rootLifecycleServiceFactory.ts index e724c2f5cb..abba4beece 100644 --- a/packages/backend-defaults/src/entrypoints/rootLifecycle/rootLifecycleServiceFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootLifecycle/rootLifecycleServiceFactory.ts @@ -65,6 +65,39 @@ export class BackendLifecycleImpl implements RootLifecycleService { ); } + #hasBeforeShutdown = false; + #beforeShutdownTasks: Array<{ hook: () => void | Promise }> = []; + + addBeforeShutdownHook(hook: () => void): void { + if (this.#hasBeforeShutdown) { + throw new Error( + 'Attempt to add before shutdown hook after shutdown has started', + ); + } + this.#beforeShutdownTasks.push({ hook }); + } + + async beforeShutdown(): Promise { + if (this.#hasBeforeShutdown) { + return; + } + this.#hasBeforeShutdown = true; + + this.logger.debug( + `Running ${this.#beforeShutdownTasks.length} before shutdown tasks...`, + ); + await Promise.all( + this.#beforeShutdownTasks.map(async ({ hook }) => { + try { + await hook(); + this.logger.debug(`Before shutdown hook succeeded`); + } catch (error) { + this.logger.error(`Before shutdown hook failed, ${error}`); + } + }), + ); + } + #hasShutdown = false; #shutdownTasks: Array<{ hook: LifecycleServiceShutdownHook; diff --git a/packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerImpl.test.ts b/packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerImpl.test.ts index 3c641feadf..1ce1401829 100644 --- a/packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerImpl.test.ts +++ b/packages/backend-defaults/src/entrypoints/scheduler/lib/PluginTaskSchedulerImpl.test.ts @@ -55,7 +55,11 @@ describe('PluginTaskManagerImpl', () => { const manager = new PluginTaskSchedulerImpl( async () => knex, mockServices.logger.mock(), - { addShutdownHook, addStartupHook: jest.fn() }, + { + addShutdownHook, + addBeforeShutdownHook: jest.fn(), + addStartupHook: jest.fn(), + }, ); return { knex, manager }; } diff --git a/packages/backend-plugin-api/report.api.md b/packages/backend-plugin-api/report.api.md index a886ba4271..8416c6c200 100644 --- a/packages/backend-plugin-api/report.api.md +++ b/packages/backend-plugin-api/report.api.md @@ -507,7 +507,10 @@ export interface RootHttpRouterService { } // @public -export interface RootLifecycleService extends LifecycleService {} +export interface RootLifecycleService extends LifecycleService { + // (undocumented) + addBeforeShutdownHook(hook: () => void | Promise): void; +} // @public export interface RootLoggerService extends LoggerService {} diff --git a/packages/backend-plugin-api/src/services/definitions/RootLifecycleService.ts b/packages/backend-plugin-api/src/services/definitions/RootLifecycleService.ts index b962fa68ba..2fc9556938 100644 --- a/packages/backend-plugin-api/src/services/definitions/RootLifecycleService.ts +++ b/packages/backend-plugin-api/src/services/definitions/RootLifecycleService.ts @@ -23,4 +23,6 @@ import { LifecycleService } from './LifecycleService'; * * @public */ -export interface RootLifecycleService extends LifecycleService {} +export interface RootLifecycleService extends LifecycleService { + addBeforeShutdownHook(hook: () => void | Promise): void; +} diff --git a/packages/backend-test-utils/src/next/services/mockServices.ts b/packages/backend-test-utils/src/next/services/mockServices.ts index a2c3b34ead..b9bca62767 100644 --- a/packages/backend-test-utils/src/next/services/mockServices.ts +++ b/packages/backend-test-utils/src/next/services/mockServices.ts @@ -472,6 +472,7 @@ export namespace mockServices { export const factory = () => rootLifecycleServiceFactory; export const mock = simpleMock(coreServices.rootLifecycle, () => ({ addShutdownHook: jest.fn(), + addBeforeShutdownHook: jest.fn(), addStartupHook: jest.fn(), })); } diff --git a/yarn.lock b/yarn.lock index 2c40b37ad1..a825ebd55a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3652,7 +3652,6 @@ __metadata: pg-format: ^1.0.4 raw-body: ^2.4.1 selfsigned: ^2.0.0 - stoppable: ^1.1.0 supertest: ^7.0.0 tar: ^6.1.12 triple-beam: ^1.4.1