Merge pull request #28621 from backstage/backend-startup-optional-config

Add configuration to permit backend startup failure

.changeset/twenty-carrots-jog.md

@@ -0,0 +1,31 @@
+---
+'@backstage/backend-app-api': patch
+---
+
+Added a configuration to permit backend plugin failures on startup:
+
+```yaml
+backend:
+  ...
+  startup:
+    plugins:
+      plugin-x:
+        onPluginBootFailure: continue
+```
+
+This configuration permits `plugin-x` to fail on startup. Omitting the
+`onPluginBootFailure` configuration matches the previous behavior, wherein any
+individual plugin failure aborts backend startup.
+
+The default can also be changed, so that continuing on failure is the default
+unless otherwise specified:
+
+```yaml
+backend:
+  startup:
+    default:
+      onPluginBootFailure: continue
+    plugins:
+      catalog:
+        onPluginBootFailure: abort
+```

packages/backend-app-api/config.d.ts

@@ -23,5 +23,31 @@ export interface Config {
           include?: string[];
           exclude?: string[];
         };
+
+    startup?: {
+      default?: {
+        /**
+         * The default value for `onPluginBootFailure` if not specified for a particular plugin.
+         * This defaults to 'abort', which means `onPluginBootFailure: continue` must be specified
+         * for backend startup to continue on plugin boot failure. This can also be set to
+         * 'continue', which flips the logic for individual plugins so that they must be set to
+         * `onPluginBootFailure: abort` to be required.
+         */
+        onPluginBootFailure?: 'continue' | 'abort';
+      };
+      plugins?: {
+        [pluginId: string]: {
+          /**
+           * Used to control backend startup behavior when this plugin fails to boot up. Setting
+           * this to `continue` allows the backend to continue starting up, even if this plugin
+           * fails. This can enable leaving a crashing plugin installed, but still permit backend
+           * startup, which may help troubleshoot data-dependent issues. Plugin failures for plugins
+           * set to `abort` are fatal (this is the default unless overridden by the `default`
+           * setting).
+           */
+          onPluginBootFailure?: 'continue' | 'abort';
+        };
+      };
+    };
   };
 }

packages/backend-app-api/src/wiring/BackendInitializer.test.ts

@@ -560,6 +560,95 @@ describe('BackendInitializer', () => {
     );
   });
 
+  it('should permit startup errors for plugins with onPluginBootFailure: continue', async () => {
+    const init = new BackendInitializer([
+      mockServices.rootLifecycle.factory(),
+      mockServices.rootLogger.factory(),
+      mockServices.rootConfig.factory({
+        data: {
+          backend: {
+            startup: { plugins: { test: { onPluginBootFailure: 'continue' } } },
+          },
+        },
+      }),
+    ]);
+    init.add(
+      createBackendPlugin({
+        pluginId: 'test',
+        register(reg) {
+          reg.registerInit({
+            deps: {},
+            async init() {
+              throw new Error('NOPE');
+            },
+          });
+        },
+      }),
+    );
+    await expect(init.start()).resolves.not.toThrow();
+  });
+
+  it('should permit startup errors if the default onPluginBootFailure is continue', async () => {
+    const init = new BackendInitializer([
+      mockServices.rootLifecycle.factory(),
+      mockServices.rootLogger.factory(),
+      mockServices.rootConfig.factory({
+        data: {
+          backend: {
+            startup: { default: { onPluginBootFailure: 'continue' } },
+          },
+        },
+      }),
+    ]);
+    init.add(
+      createBackendPlugin({
+        pluginId: 'test',
+        register(reg) {
+          reg.registerInit({
+            deps: {},
+            async init() {
+              throw new Error('NOPE');
+            },
+          });
+        },
+      }),
+    );
+    await expect(init.start()).resolves.not.toThrow();
+  });
+
+  it('should forward errors for plugins explicitly marked to abort when the default is continue', async () => {
+    const init = new BackendInitializer([
+      mockServices.rootLifecycle.factory(),
+      mockServices.rootLogger.factory(),
+      mockServices.rootConfig.factory({
+        data: {
+          backend: {
+            startup: {
+              default: { onPluginBootFailure: 'continue' },
+              plugins: { test: { onPluginBootFailure: 'abort' } },
+            },
+          },
+        },
+      }),
+    ]);
+    init.add(
+      createBackendPlugin({
+        pluginId: 'test',
+        register(reg) {
+          reg.registerInit({
+            deps: {},
+            async init() {
+              throw new Error('NOPE');
+            },
+          });
+        },
+      }),
+    );
+    await expect(init.start()).rejects.toThrow(
+      "Plugin 'test' startup failed; caused by Error: NOPE",
+    );
+  });
+
   it('should forward errors when multiple plugins fail to start', async () => {
     const init = new BackendInitializer([]);
     init.add(

packages/backend-app-api/src/wiring/BackendInitializer.ts

@@ -24,6 +24,7 @@ import {
   RootLifecycleService,
   createServiceFactory,
 } from '@backstage/backend-plugin-api';
+import { Config } from '@backstage/config';
 import { ServiceOrExtensionPoint } from './types';
 // Direct internal import to avoid duplication
 // eslint-disable-next-line @backstage/no-relative-monorepo-imports
@@ -321,9 +322,19 @@ export class BackendInitializer {
       await this.#serviceRegistry.get(coreServices.rootLogger, 'root'),
     );
 
+    const rootConfig = await this.#serviceRegistry.get(
+      coreServices.rootConfig,
+      'root',
+    );
+
     // All plugins are initialized in parallel
     const results = await Promise.allSettled(
       allPluginIds.map(async pluginId => {
+        const isBootFailurePermitted = this.#getPluginBootFailurePredicate(
+          pluginId,
+          rootConfig,
+        );
+
         try {
           // Initialize all eager services
           await this.#serviceRegistry.initializeEagerServicesWithScope(
@@ -392,8 +403,12 @@ export class BackendInitializer {
           await lifecycleService.startup();
         } catch (error: unknown) {
           assertError(error);
-          initLogger.onPluginFailed(pluginId, error);
-          throw error;
+          if (isBootFailurePermitted) {
+            initLogger.onPermittedPluginFailure(pluginId, error);
+          } else {
+            initLogger.onPluginFailed(pluginId, error);
+            throw error;
+          }
         }
       }),
     );
@@ -620,6 +635,20 @@ export class BackendInitializer {
       }
     }
   }
+
+  #getPluginBootFailurePredicate(pluginId: string, config?: Config): boolean {
+    const defaultStartupBootFailureValue =
+      config?.getOptionalString(
+        'backend.startup.default.onPluginBootFailure',
+      ) ?? 'abort';
+
+    const pluginStartupBootFailureValue =
+      config?.getOptionalString(
+        `backend.startup.plugins.${pluginId}.onPluginBootFailure`,
+      ) ?? defaultStartupBootFailureValue;
+
+    return pluginStartupBootFailureValue === 'continue';
+  }
 }
 
 function toInternalBackendFeature(

packages/backend-app-api/src/wiring/createInitializationLogger.ts

@@ -28,6 +28,7 @@ export function createInitializationLogger(
 ): {
   onPluginStarted(pluginId: string): void;
   onPluginFailed(pluginId: string, error: Error): void;
+  onPermittedPluginFailure(pluginId: string, error: Error): void;
   onAllStarted(): void;
 } {
   const logger = rootLogger?.child({ type: 'initialization' });
@@ -79,6 +80,13 @@ export function createInitializationLogger(
         error,
       );
     },
+    onPermittedPluginFailure(pluginId: string, error: Error) {
+      starting.delete(pluginId);
+      logger?.error(
+        `Plugin '${pluginId}' threw an error during startup, but boot failure is permitted for this plugin so startup will continue.`,
+        error,
+      );
+    },
     onAllStarted() {
       logger?.info(`Plugin initialization complete${getInitStatus()}`);