Merge pull request #27961 from backstage/camilaibs/create-accepting-request-health-check-endpoint

Stop accepting traffic before shutdown
2024-12-10 13:21:46 +01:00
parent 2206b608a9 ba082fd507
commit 84407de448
26 changed files with 541 additions and 30 deletions
@@ -0,0 +1,7 @@
+---
+'@backstage/backend-plugin-api': patch
+---
+
+The `RootLifecycleService` now has a new `addBeforeShutdownHook` method, and hooks added through this method will run immediately when a termination event is received.
+
+The backend will not proceed with the shutdown and run the `Shutdown` hooks until all `BeforeShutdown` hooks have completed.
@@ -0,0 +1,5 @@
+---
+'@backstage/backend-test-utils': patch
+---
+
+Mock the new `RootLifecycleService.addBeforeShutdownHook` method.
@@ -0,0 +1,5 @@
+---
+'@backstage/backend-defaults': patch
+---
+
+Implements the `DefaultRootLifecycleService.addBeforeShutdownHook` method, and updates `DefaultRootHttpRouterService` and `DefaultRootHealthService` to listen to that event to stop accepting traffic and close service connections.
@@ -0,0 +1,6 @@
+---
+'@backstage/backend-defaults': patch
+---
+
+Remove use of the `stoppable` library on the `DefaultRootHttpRouterService` as Node's native http server [close](https://nodejs.org/api/http.html#serverclosecallback) method already drains requests.
+Also, we pass a new `lifecycleMiddleware` to the `rootHttpRouterServiceFactory` configure function that must be called manually if you don't call `applyDefaults`.
@@ -0,0 +1,5 @@
+---
+'@backstage/backend-app-api': patch
+---
+
+As soon as a backend termination signal is received, call before shutting down root lifecycle hooks.
@@ -42,6 +42,29 @@ createBackendPlugin({

 ## Configuring the service

+### Via `app-config.yaml`
+
+The `app-config.yaml` file provides configurable options that can be adjusted to meet your `RootHttpRouterService` specific requirements:
+
+```yaml
+backend:
+  lifecycle:
+    # (Optional) The maximum time that paused requests will wait for the service to start, before returning an error (defaults to 5 seconds).
+    # Supported formats:
+    # - A string in the format of '1d', '2 seconds' etc. as supported by the `ms` library.
+    # - A standard ISO formatted duration string, e.g. 'P2DT6H' or 'PT1M'.
+    # - An object with individual units (in plural) as keys, e.g. `{ days: 2, hours: 6 }`.
+    startupRequestPauseTimeout: { seconds: 10 }
+    # (Optional) The minimum time that the HTTP server will delay the shutdown of the backend. During this delay health checks will be set to failing, allowing traffic to drain (defaults to 0 seconds).
+    # Supported formats:
+    # - A string in the format of '1d', '2 seconds' etc. as supported by the `ms` library.
+    # - A standard ISO formatted duration string, e.g. 'P2DT6H' or 'PT1M'.
+    # - An object with individual units (in plural) as keys, e.g. `{ days: 2, hours: 6 }`.
+    serverShutdownTimeout: { seconds: 20 }
+```
+
+### Via Code
+
 There's additional options that you can pass to configure the root HTTP Router service. These options are passed when you call `createBackend`.

 - `indexPath` - optional path to forward all unmatched requests to. Defaults to `/api/app` which is the `app-backend` plugin responsible for serving the frontend application through the backend.
@@ -449,6 +449,11 @@ export class BackendInitializer {
      // The startup failed, but we may still want to do cleanup so we continue silently
    }

+    const rootLifecycleService = await this.#getRootLifecycleImpl();
+
+    // Root services like the health one need to immediatelly be notified of the shutdown
+    await rootLifecycleService.beforeShutdown();
+
    // Get all plugins.
    const allPlugins = new Set<string>();
    for (const feature of this.#registrations) {
@@ -468,14 +473,14 @@ export class BackendInitializer {
    );

    // Once all plugin shutdown hooks are done, run root shutdown hooks.
-    const lifecycleService = await this.#getRootLifecycleImpl();
-    await lifecycleService.shutdown();
+    await rootLifecycleService.shutdown();
  }

  // Bit of a hacky way to grab the lifecycle services, potentially find a nicer way to do this
  async #getRootLifecycleImpl(): Promise<
    RootLifecycleService & {
      startup(): Promise<void>;
+      beforeShutdown(): Promise<void>;
      shutdown(): Promise<void>;
    }
  > {
@@ -36,6 +36,7 @@ describe('createSpecializedBackend', () => {
            deps: {},
            factory: async () => ({
              addStartupHook: () => {},
+              addBeforeShutdownHook: () => {},
              addShutdownHook: () => {},
            }),
          }),
@@ -44,6 +45,7 @@ describe('createSpecializedBackend', () => {
            deps: {},
            factory: async () => ({
              addStartupHook: () => {},
+              addBeforeShutdownHook: () => {},
              addShutdownHook: () => {},
            }),
          }),
@@ -28,6 +28,29 @@ export interface Config {
     */
    baseUrl: string;

+    lifecycle?: {
+      /**
+       * The maximum time that paused requests will wait for the service to start, before returning an error.
+       * Defaults to 5 seconds
+       * Supported formats:
+       * - A string in the format of '1d', '2 seconds' etc. as supported by the `ms`
+       *   library.
+       * - A standard ISO formatted duration string, e.g. 'P2DT6H' or 'PT1M'.
+       * - An object with individual units (in plural) as keys, e.g. `{ days: 2, hours: 6 }`.
+       */
+      startupRequestPauseTimeout?: string | HumanDuration;
+      /**
+       * The minimum time that the HTTP server will delay the shutdown of the backend. During this delay health checks will be set to failing, allowing traffic to drain.
+       * Defaults to 0 seconds.
+       * Supported formats:
+       * - A string in the format of '1d', '2 seconds' etc. as supported by the `ms`
+       *   library.
+       * - A standard ISO formatted duration string, e.g. 'P2DT6H' or 'PT1M'.
+       * - An object with individual units (in plural) as keys, e.g. `{ days: 2, hours: 6 }`.
+       */
+      serverShutdownDelay?: string | HumanDuration;
+    };
+
    /** Address that the backend should listen to. */
    listen?:
      | string
@@ -175,7 +175,6 @@
    "pg-format": "^1.0.4",
    "raw-body": "^2.4.1",
    "selfsigned": "^2.0.0",
-    "stoppable": "^1.1.0",
    "tar": "^6.1.12",
    "triple-beam": "^1.4.1",
    "uuid": "^11.0.0",
@@ -134,6 +134,8 @@ export interface RootHttpRouterConfigureContext {
  // (undocumented)
  lifecycle: LifecycleService;
  // (undocumented)
+  lifecycleMiddleware: RequestHandler;
+  // (undocumented)
  logger: LoggerService;
  // (undocumented)
  middleware: MiddlewareFactory;
@@ -47,6 +47,7 @@ describe('createBackend', () => {
        deps: {},
        factory: async () => ({
          addStartupHook: () => {},
+          addBeforeShutdownHook: () => {},
          addShutdownHook: () => {},
        }),
      }),
@@ -57,6 +58,7 @@ describe('createBackend', () => {
        deps: {},
        factory: async () => ({
          addStartupHook: () => {},
+          addBeforeShutdownHook: () => {},
          addShutdownHook: () => {},
        }),
      }),
@@ -22,7 +22,6 @@ import {
  HttpRouterServiceAuthPolicy,
 } from '@backstage/backend-plugin-api';
 import {
-  createLifecycleMiddleware,
  createCookieAuthRefreshMiddleware,
  createCredentialsBarrier,
  createAuthIntegrationRouter,
@@ -43,12 +42,11 @@ export const httpRouterServiceFactory = createServiceFactory({
  deps: {
    plugin: coreServices.pluginMetadata,
    config: coreServices.rootConfig,
-    lifecycle: coreServices.lifecycle,
    rootHttpRouter: coreServices.rootHttpRouter,
    auth: coreServices.auth,
    httpAuth: coreServices.httpAuth,
  },
-  async factory({ auth, httpAuth, config, plugin, rootHttpRouter, lifecycle }) {
+  async factory({ auth, httpAuth, config, plugin, rootHttpRouter }) {
    const router = PromiseRouter();

    rootHttpRouter.use(`/api/${plugin.getId()}`, router);
@@ -59,7 +57,6 @@ export const httpRouterServiceFactory = createServiceFactory({
    });

    router.use(createAuthIntegrationRouter({ auth }));
-    router.use(createLifecycleMiddleware({ lifecycle }));
    router.use(credentialsBarrier.middleware);
    router.use(createCookieAuthRefreshMiddleware({ auth, httpAuth }));

@@ -53,11 +53,11 @@ describe('DefaultRootHealthService', () => {

    it(`should return a 500 response if the server has stopped`, async () => {
      let mockServerStartedFn = () => {};
-      let mockServerStoppedFn = () => {};
+      let mockServerBeforeStoppedFn = () => {};

      const lifecycle = mockServices.rootLifecycle.mock({
        addStartupHook: jest.fn(fn => (mockServerStartedFn = fn)),
-        addShutdownHook: jest.fn(fn => (mockServerStoppedFn = fn)),
+        addBeforeShutdownHook: jest.fn(fn => (mockServerBeforeStoppedFn = fn)),
      });

      const service = new DefaultRootHealthService({
@@ -65,11 +65,11 @@ describe('DefaultRootHealthService', () => {
      });

      mockServerStartedFn();
-      mockServerStoppedFn();
+      mockServerBeforeStoppedFn();
      await expect(service.getReadiness()).resolves.toEqual({
        status: 503,
        payload: {
-          message: 'Backend has not started yet',
+          message: 'Backend is shuttting down',
          status: 'error',
        },
      });
@@ -23,14 +23,14 @@ import {

 /** @internal */
 export class DefaultRootHealthService implements RootHealthService {
-  #isRunning = false;
+  #state: 'init' | 'up' | 'down' = 'init';

  constructor(readonly options: { lifecycle: RootLifecycleService }) {
    options.lifecycle.addStartupHook(() => {
-      this.#isRunning = true;
+      this.#state = 'up';
    });
-    options.lifecycle.addShutdownHook(() => {
-      this.#isRunning = false;
+    options.lifecycle.addBeforeShutdownHook(() => {
+      this.#state = 'down';
    });
  }

@@ -39,12 +39,18 @@ export class DefaultRootHealthService implements RootHealthService {
  }

  async getReadiness(): Promise<{ status: number; payload?: any }> {
-    if (!this.#isRunning) {
+    if (this.#state === 'init') {
      return {
        status: 503,
        payload: { message: 'Backend has not started yet', status: 'error' },
      };
    }
+    if (this.#state === 'down') {
+      return {
+        status: 503,
+        payload: { message: 'Backend is shuttting down', status: 'error' },
+      };
+    }

    return { status: 200, payload: { status: 'ok' } };
  }
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createLifecycleMiddleware } from './createLifecycleMiddleware';
+import { BackendLifecycleImpl } from '../rootLifecycle/rootLifecycleServiceFactory';
+import { mockServices } from '@backstage/backend-test-utils';
+import { ServiceUnavailableError } from '@backstage/errors';
+
+describe('createLifecycleMiddleware', () => {
+  it('should pause requests when plugin is not ready', async () => {
+    const lifecycle = new BackendLifecycleImpl(mockServices.rootLogger());
+
+    const middleware = createLifecycleMiddleware({ lifecycle });
+
+    const next = jest.fn();
+    middleware({} as any, {} as any, next);
+    expect(next).not.toHaveBeenCalled();
+    await lifecycle.startup();
+
+    // pending call
+    expect(next).toHaveBeenCalledWith();
+
+    // new call
+    const next2 = jest.fn();
+    middleware({} as any, {} as any, next2);
+    expect(next2).toHaveBeenCalledWith();
+  });
+
+  it('should throw ServiceUnavailableError after shutdown', async () => {
+    const lifecycle = new BackendLifecycleImpl(mockServices.rootLogger());
+    const middleware = createLifecycleMiddleware({ lifecycle });
+
+    const next = jest.fn();
+    middleware({} as any, {} as any, next);
+    expect(next).not.toHaveBeenCalled();
+    await lifecycle.shutdown();
+
+    // pending call
+    expect(next).toHaveBeenCalledWith(
+      new ServiceUnavailableError('Service is shutting down'),
+    );
+
+    // new call
+    const next2 = jest.fn();
+    middleware({} as any, {} as any, next2);
+    expect(next2).toHaveBeenCalledWith(
+      new ServiceUnavailableError('Service is shutting down'),
+    );
+  });
+
+  it('should throw ServiceUnavailableError after timeout', async () => {
+    const lifecycle = new BackendLifecycleImpl(mockServices.rootLogger());
+    const middleware = createLifecycleMiddleware({
+      lifecycle,
+      startupRequestPauseTimeout: { milliseconds: 1 },
+    });
+
+    const next = jest.fn();
+    middleware({} as any, {} as any, next);
+    expect(next).not.toHaveBeenCalled();
+
+    await new Promise(r => setTimeout(r, 2));
+
+    expect(next).toHaveBeenCalledWith(
+      new ServiceUnavailableError('Service has not started up yet'),
+    );
+  });
+
+  it('should delay service shutdown for the default timeout duration', async () => {
+    jest.useFakeTimers();
+    const defaultTimeout = 30000;
+    const lifecycle = new BackendLifecycleImpl(mockServices.rootLogger());
+    createLifecycleMiddleware({ lifecycle });
+    const beforeShutdownPromise = lifecycle.beforeShutdown().then(() => {
+      jest.useRealTimers();
+    });
+    jest.advanceTimersByTime(defaultTimeout);
+    return expect(beforeShutdownPromise).resolves.toBeUndefined();
+  });
+
+  it('should delay service shutdown for the configured timeout duration - time in human duration', async () => {
+    jest.useFakeTimers();
+    const configuredTimeout = 20000;
+    const lifecycle = new BackendLifecycleImpl(mockServices.rootLogger());
+    createLifecycleMiddleware({
+      lifecycle,
+      serverShutdownDelay: { milliseconds: configuredTimeout },
+    });
+    const beforeShutdownPromise = lifecycle.beforeShutdown().then(() => {
+      jest.useRealTimers();
+    });
+    jest.advanceTimersByTime(configuredTimeout);
+    return expect(beforeShutdownPromise).resolves.toBeUndefined();
+  });
+});
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { RootLifecycleService } from '@backstage/backend-plugin-api';
+import { ServiceUnavailableError } from '@backstage/errors';
+import { HumanDuration, durationToMilliseconds } from '@backstage/types';
+import { RequestHandler } from 'express';
+
+export const DEFAULT_STARTUP_REQUEST_PAUSE_TIMEOUT = { seconds: 5 };
+export const DEFAULT_SERVER_SHUTDOWN_TIMEOUT = { seconds: 0 };
+
+/**
+ * Options for {@link createLifecycleMiddleware}.
+ * @public
+ */
+export interface LifecycleMiddlewareOptions {
+  lifecycle: RootLifecycleService;
+  /**
+   * The maximum time that paused requests will wait for the service to start, before returning an error.
+   *
+   * Defaults to 5 seconds.
+   */
+  startupRequestPauseTimeout?: HumanDuration;
+  /**
+   * The maximum time that the server will wait for stop accepting traffic, before returning an error.
+   *
+   * Defaults to 0 seconds.
+   */
+  serverShutdownDelay?: HumanDuration;
+}
+
+/**
+ * Creates a middleware that pauses requests until the service has started.
+ *
+ * @remarks
+ *
+ * Requests that arrive before the service has started will be paused until startup is complete.
+ * If the service does not start within the provided timeout, the request will be rejected with a
+ * {@link @backstage/errors#ServiceUnavailableError}.
+ *
+ * If the service is shutting down, all requests will be rejected with a
+ * {@link @backstage/errors#ServiceUnavailableError}.
+ *
+ * @public
+ */
+export function createLifecycleMiddleware(
+  options: LifecycleMiddlewareOptions,
+): RequestHandler {
+  const { lifecycle, startupRequestPauseTimeout, serverShutdownDelay } =
+    options;
+
+  let state: 'init' | 'up' | 'down' = 'init';
+  const waiting = new Set<{
+    next: (err?: Error) => void;
+    timeout: NodeJS.Timeout;
+  }>();
+
+  lifecycle.addStartupHook(async () => {
+    if (state === 'init') {
+      state = 'up';
+      for (const item of waiting) {
+        clearTimeout(item.timeout);
+        item.next();
+      }
+      waiting.clear();
+    }
+  });
+
+  lifecycle.addBeforeShutdownHook(async () => {
+    const timeoutMs = durationToMilliseconds(
+      serverShutdownDelay ?? DEFAULT_SERVER_SHUTDOWN_TIMEOUT,
+    );
+    return await new Promise(resolve => {
+      setTimeout(resolve, timeoutMs);
+    });
+  });
+
+  lifecycle.addShutdownHook(async () => {
+    state = 'down';
+    for (const item of waiting) {
+      clearTimeout(item.timeout);
+      item.next(new ServiceUnavailableError('Service is shutting down'));
+    }
+    waiting.clear();
+  });
+
+  return (_req, _res, next) => {
+    if (state === 'up') {
+      next();
+      return;
+    } else if (state === 'down') {
+      next(new ServiceUnavailableError('Service is shutting down'));
+      return;
+    }
+
+    const timeoutMs = durationToMilliseconds(
+      startupRequestPauseTimeout ?? DEFAULT_STARTUP_REQUEST_PAUSE_TIMEOUT,
+    );
+
+    const item = {
+      next,
+      timeout: setTimeout(() => {
+        if (waiting.delete(item)) {
+          next(new ServiceUnavailableError('Service has not started up yet'));
+        }
+      }, timeoutMs),
+    };
+
+    waiting.add(item);
+  };
+}
@@ -16,7 +16,6 @@

 import * as http from 'http';
 import * as https from 'https';
-import stoppableServer from 'stoppable';
 import { RequestListener } from 'http';
 import { LoggerService } from '@backstage/backend-plugin-api';
 import { HttpServerOptions, ExtendedHttpServer } from './types';
@@ -33,13 +32,6 @@ export async function createHttpServer(
  deps: { logger: LoggerService },
 ): Promise<ExtendedHttpServer> {
  const server = await createServer(listener, options, deps);
-
-  const stopper = stoppableServer(server, 0);
-  // The stopper here is actually the server itself, so if we try
-  // to call stopper.stop() down in the stop implementation, we'll
-  // be calling ourselves.
-  const stopServer = stopper.stop.bind(stopper);
-
  return Object.assign(server, {
    start() {
      return new Promise<void>((resolve, reject) => {
@@ -61,7 +53,7 @@ export async function createHttpServer(

    stop() {
      return new Promise<void>((resolve, reject) => {
-        stopServer((error?: Error) => {
+        server.close(error => {
          if (error) {
            reject(error);
          } else {
@@ -21,7 +21,12 @@ import {
 import { Express } from 'express';
 import request from 'supertest';
 import { rootHttpRouterServiceFactory } from './rootHttpRouterServiceFactory';
-import { ServiceFactory, coreServices } from '@backstage/backend-plugin-api';
+import {
+  ServiceFactory,
+  coreServices,
+  createServiceFactory,
+} from '@backstage/backend-plugin-api';
+import { BackendLifecycleImpl } from '../rootLifecycle/rootLifecycleServiceFactory';

 async function createExpressApp(...dependencies: ServiceFactory[]) {
  let app: Express | undefined = undefined;
@@ -198,4 +203,130 @@ describe('rootHttpRouterServiceFactory', () => {
      'Invalid header value in at backend.health.headers, must be a non-empty string',
    );
  });
+
+  it('should wait the server to shutdown', async () => {
+    jest.useFakeTimers();
+
+    let app: Express | undefined = undefined;
+    const lifecycleMock = new BackendLifecycleImpl(mockServices.rootLogger());
+
+    const tester = ServiceFactoryTester.from(
+      rootHttpRouterServiceFactory({
+        configure(options) {
+          console.log('configure');
+          options.app.use(options.healthRouter);
+          options.app.use(options.lifecycleMiddleware);
+          options.app.get('/test', (_req, res) => {
+            res.status(200).send({ status: 'ok' }).end();
+          });
+          options.app.use(options.middleware.error());
+          app = options.app;
+        },
+      }),
+      {
+        dependencies: [
+          mockServices.rootConfig.factory({
+            data: {
+              app: { baseUrl: 'http://localhost' },
+              backend: {
+                baseUrl: 'http://localhost',
+                listen: { host: '', port: 0 },
+                lifecycle: {
+                  serverShutdownDelay: '30s',
+                },
+              },
+            },
+          }),
+          createServiceFactory({
+            service: coreServices.rootLifecycle,
+            deps: {},
+            factory() {
+              return lifecycleMock;
+            },
+          }),
+        ],
+      },
+    );
+
+    await tester.getSubject();
+
+    // Trigger creation of the http service, accessing the app instance through the configure callback
+    const lifecycle = await tester.getService(coreServices.rootLifecycle);
+
+    await (lifecycle as any).startup(); // Trigger startup by calling the private startup method
+
+    await request(app!).get('/test').expect(200, {
+      status: 'ok',
+    });
+
+    await request(app)
+      .get('/.backstage/health/v1/liveness')
+      .expect(200, { status: 'ok' });
+
+    await request(app).get('/.backstage/health/v1/readiness').expect(200, {
+      status: 'ok',
+    });
+
+    const beforeShutdownPromise = (lifecycle as any)
+      .beforeShutdown()
+      .then(() => {
+        return (lifecycle as any).shutdown();
+      });
+
+    // Continue accepting requests
+    await request(app!).get('/test').expect(200, {
+      status: 'ok',
+    });
+
+    await request(app)
+      .get('/.backstage/health/v1/liveness')
+      .expect(200, { status: 'ok' });
+
+    // Immediately start failing the readiness health check
+    await request(app).get('/.backstage/health/v1/readiness').expect(503, {
+      message: 'Backend is shuttting down',
+      status: 'error',
+    });
+
+    jest.advanceTimersByTime(29999);
+
+    // Still accepting requests 1 ms before shutdown
+    await request(app!).get('/test').expect(200, {
+      status: 'ok',
+    });
+
+    await request(app)
+      .get('/.backstage/health/v1/liveness')
+      .expect(200, { status: 'ok' });
+
+    await request(app).get('/.backstage/health/v1/readiness').expect(503, {
+      message: 'Backend is shuttting down',
+      status: 'error',
+    });
+
+    jest.advanceTimersByTime(1);
+
+    // No longer accepting requests after shutdown
+    await request(app!)
+      .get('/test')
+      .expect(503, {
+        error: {
+          name: 'ServiceUnavailableError',
+          message: 'Service is shutting down',
+        },
+        request: { method: 'GET', url: '/test' },
+        response: { statusCode: 503 },
+      });
+
+    await request(app)
+      .get('/.backstage/health/v1/liveness')
+      .expect(200, { status: 'ok' });
+
+    await request(app).get('/.backstage/health/v1/readiness').expect(503, {
+      message: 'Backend is shuttting down',
+      status: 'error',
+    });
+
+    return await expect(beforeShutdownPromise).resolves.toBeUndefined();
+  });
 });
@@ -30,6 +30,9 @@ import {
 } from './http';
 import { DefaultRootHttpRouter } from './DefaultRootHttpRouter';
 import { createHealthRouter } from './createHealthRouter';
+import { createLifecycleMiddleware } from './createLifecycleMiddleware';
+import { readDurationFromConfig } from '@backstage/config';
+import { HumanDuration } from '@backstage/types';

 /**
 * @public
@@ -43,6 +46,7 @@ export interface RootHttpRouterConfigureContext {
  logger: LoggerService;
  lifecycle: LifecycleService;
  healthRouter: RequestHandler;
+  lifecycleMiddleware: RequestHandler;
  applyDefaults: () => void;
 }

@@ -90,6 +94,27 @@ const rootHttpRouterServiceFactoryWithOptions = (
      const routes = router.handler();

      const healthRouter = createHealthRouter({ config, health });
+
+      let startupRequestPauseTimeout: HumanDuration | undefined;
+      if (config.has('backend.lifecycle.startupRequestPauseTimeout')) {
+        startupRequestPauseTimeout = readDurationFromConfig(config, {
+          key: 'backend.lifecycle.startupRequestPauseTimeout',
+        });
+      }
+
+      let serverShutdownDelay: HumanDuration | undefined;
+      if (config.has('backend.lifecycle.serverShutdownDelay')) {
+        serverShutdownDelay = readDurationFromConfig(config, {
+          key: 'backend.lifecycle.serverShutdownDelay',
+        });
+      }
+
+      const lifecycleMiddleware = createLifecycleMiddleware({
+        lifecycle,
+        startupRequestPauseTimeout,
+        serverShutdownDelay,
+      });
+
      const server = await createHttpServer(
        app,
        readHttpServerOptions(config.getOptionalConfig('backend')),
@@ -105,6 +130,7 @@ const rootHttpRouterServiceFactoryWithOptions = (
        logger,
        lifecycle,
        healthRouter,
+        lifecycleMiddleware,
        applyDefaults() {
          if (process.env.NODE_ENV === 'development') {
            app.set('json spaces', 2);
@@ -114,6 +140,7 @@ const rootHttpRouterServiceFactoryWithOptions = (
          app.use(middleware.compression());
          app.use(middleware.logging());
          app.use(healthRouter);
+          app.use(lifecycleMiddleware);
          app.use(routes);
          app.use(middleware.notFound());
          app.use(middleware.error());
@@ -65,6 +65,39 @@ export class BackendLifecycleImpl implements RootLifecycleService {
    );
  }

+  #hasBeforeShutdown = false;
+  #beforeShutdownTasks: Array<{ hook: () => void | Promise<void> }> = [];
+
+  addBeforeShutdownHook(hook: () => void): void {
+    if (this.#hasBeforeShutdown) {
+      throw new Error(
+        'Attempt to add before shutdown hook after shutdown has started',
+      );
+    }
+    this.#beforeShutdownTasks.push({ hook });
+  }
+
+  async beforeShutdown(): Promise<void> {
+    if (this.#hasBeforeShutdown) {
+      return;
+    }
+    this.#hasBeforeShutdown = true;
+
+    this.logger.debug(
+      `Running ${this.#beforeShutdownTasks.length} before shutdown tasks...`,
+    );
+    await Promise.all(
+      this.#beforeShutdownTasks.map(async ({ hook }) => {
+        try {
+          await hook();
+          this.logger.debug(`Before shutdown hook succeeded`);
+        } catch (error) {
+          this.logger.error(`Before shutdown hook failed, ${error}`);
+        }
+      }),
+    );
+  }
+
  #hasShutdown = false;
  #shutdownTasks: Array<{
    hook: LifecycleServiceShutdownHook;
@@ -55,7 +55,11 @@ describe('PluginTaskManagerImpl', () => {
    const manager = new PluginTaskSchedulerImpl(
      async () => knex,
      mockServices.logger.mock(),
-      { addShutdownHook, addStartupHook: jest.fn() },
+      {
+        addShutdownHook,
+        addBeforeShutdownHook: jest.fn(),
+        addStartupHook: jest.fn(),
+      },
    );
    return { knex, manager };
  }
@@ -507,7 +507,10 @@ export interface RootHttpRouterService {
 }

 // @public
-export interface RootLifecycleService extends LifecycleService {}
+export interface RootLifecycleService extends LifecycleService {
+  // (undocumented)
+  addBeforeShutdownHook(hook: () => void | Promise<void>): void;
+}

 // @public
 export interface RootLoggerService extends LoggerService {}
@@ -23,4 +23,6 @@ import { LifecycleService } from './LifecycleService';
 *
 * @public
 */
-export interface RootLifecycleService extends LifecycleService {}
+export interface RootLifecycleService extends LifecycleService {
+  addBeforeShutdownHook(hook: () => void | Promise<void>): void;
+}
@@ -472,6 +472,7 @@ export namespace mockServices {
    export const factory = () => rootLifecycleServiceFactory;
    export const mock = simpleMock(coreServices.rootLifecycle, () => ({
      addShutdownHook: jest.fn(),
+      addBeforeShutdownHook: jest.fn(),
      addStartupHook: jest.fn(),
    }));
  }
@@ -3652,7 +3652,6 @@ __metadata:
    pg-format: ^1.0.4
    raw-body: ^2.4.1
    selfsigned: ^2.0.0
-    stoppable: ^1.1.0
    supertest: ^7.0.0
    tar: ^6.1.12
    triple-beam: ^1.4.1