From 729a7d6e76556f250e370379fcfc56479d4bf249 Mon Sep 17 00:00:00 2001 From: "Avery, Mark" Date: Sat, 22 Mar 2025 13:57:29 -0700 Subject: [PATCH] feat(backend-app-api): add config to permit module failures on startup to not effect the plugin from starting up Signed-off-by: Avery, Mark --- .changeset/eight-toys-feel.md | 35 +++++ packages/backend-app-api/config.d.ts | 21 +++ .../src/wiring/BackendInitializer.test.ts | 135 +++++++++++++++++- .../src/wiring/BackendInitializer.ts | 59 ++++++-- .../src/wiring/createInitializationLogger.ts | 26 ++++ 5 files changed, 259 insertions(+), 17 deletions(-) create mode 100644 .changeset/eight-toys-feel.md diff --git a/.changeset/eight-toys-feel.md b/.changeset/eight-toys-feel.md new file mode 100644 index 0000000000..c0f5a12c40 --- /dev/null +++ b/.changeset/eight-toys-feel.md @@ -0,0 +1,35 @@ +--- +'@backstage/backend-app-api': patch +--- + +Added a configuration to permit backend plugin module failures on startup: + +```yaml +backend: + ... + startup: + plugins: + plugin-x: + modules: + module-y: + onPluginModuleBootFailure: continue +``` + +This configuration permits `plugin-x` with `module-y` to fail on startup. Omitting the +`onPluginModuleBootFailure` configuration matches the previous behavior, wherein any +individual plugin module failure is forwarded to the plugin and aborts backend startup. + +The default can also be changed, so that continuing on failure is the default +unless otherwise specified: + +```yaml +backend: + startup: + default: + onPluginModuleBootFailure: continue + plugins: + catalog: + modules: + github: + onPluginModuleBootFailure: abort +``` diff --git a/packages/backend-app-api/config.d.ts b/packages/backend-app-api/config.d.ts index 0cf91852a1..58511c91fb 100644 --- a/packages/backend-app-api/config.d.ts +++ b/packages/backend-app-api/config.d.ts @@ -34,6 +34,14 @@ export interface Config { * `onPluginBootFailure: abort` to be required. */ onPluginBootFailure?: 'continue' | 'abort'; + /** + * The default value for `onPluginModuleBootFailure` if not specified for a particular plugin module. + * This defaults to 'abort', which means `onPluginModuleBootFailure: continue` must be specified + * for backend startup to continue on plugin module boot failure. This can also be set to + * 'continue', which flips the logic for individual plugin modules so that they must be set to + * `onPluginModuleBootFailure: abort` to be required. + */ + onPluginModuleBootFailure?: 'continue' | 'abort'; }; plugins?: { [pluginId: string]: { @@ -46,6 +54,19 @@ export interface Config { * setting). */ onPluginBootFailure?: 'continue' | 'abort'; + modules?: { + [moduleId: string]: { + /** + * Used to control backend startup behavior when this plugin module fails to boot up. Setting + * this to `continue` allows the backend to continue starting up, even if this plugin + * module fails. This can enable leaving a crashing plugin installed, but still permit backend + * startup, which may help troubleshoot data-dependent issues. Plugin module failures for plugin modules + * set to `abort` are fatal (this is the default unless overridden by the `default` + * setting). + */ + onPluginModuleBootFailure?: 'continue' | 'abort'; + }; + }; }; }; }; diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.test.ts b/packages/backend-app-api/src/wiring/BackendInitializer.test.ts index fe0a4f526b..3b0596cd13 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.test.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.test.ts @@ -541,7 +541,7 @@ describe('BackendInitializer', () => { }); it('should forward errors when plugins fail to start', async () => { - const init = new BackendInitializer([]); + const init = new BackendInitializer(baseFactories); init.add( createBackendPlugin({ pluginId: 'test', @@ -562,8 +562,7 @@ describe('BackendInitializer', () => { it('should permit startup errors for plugins with onPluginBootFailure: continue', async () => { const init = new BackendInitializer([ - mockServices.rootLifecycle.factory(), - mockServices.rootLogger.factory(), + ...baseFactories, mockServices.rootConfig.factory({ data: { backend: { @@ -590,8 +589,7 @@ describe('BackendInitializer', () => { it('should permit startup errors if the default onPluginBootFailure is continue', async () => { const init = new BackendInitializer([ - mockServices.rootLifecycle.factory(), - mockServices.rootLogger.factory(), + ...baseFactories, mockServices.rootConfig.factory({ data: { backend: { @@ -618,8 +616,7 @@ describe('BackendInitializer', () => { it('should forward errors for plugins explicitly marked to abort when the default is continue', async () => { const init = new BackendInitializer([ - mockServices.rootLifecycle.factory(), - mockServices.rootLogger.factory(), + ...baseFactories, mockServices.rootConfig.factory({ data: { backend: { @@ -649,6 +646,130 @@ describe('BackendInitializer', () => { ); }); + it('should forward errors when plugin modules fail to start', async () => { + const init = new BackendInitializer(baseFactories); + init.add(testPlugin); + init.add( + createBackendModule({ + pluginId: 'test', + moduleId: 'mod', + register(reg) { + reg.registerInit({ + deps: {}, + async init() { + throw new Error('NOPE'); + }, + }); + }, + }), + ); + await expect(init.start()).rejects.toThrow( + "Module 'mod' for plugin 'test' startup failed; caused by Error: NOPE", + ); + }); + + it('should permit startup errors for plugin modules with onPluginModuleBootFailure: continue', async () => { + const init = new BackendInitializer([ + ...baseFactories, + mockServices.rootConfig.factory({ + data: { + backend: { + startup: { + plugins: { + test: { + modules: { mod: { onPluginModuleBootFailure: 'continue' } }, + }, + }, + }, + }, + }, + }), + ]); + init.add(testPlugin); + init.add( + createBackendModule({ + pluginId: 'test', + moduleId: 'mod', + register(reg) { + reg.registerInit({ + deps: {}, + async init() { + throw new Error('NOPE'); + }, + }); + }, + }), + ); + await expect(init.start()).resolves.not.toThrow(); + }); + + it('should permit startup errors if the default onPluginModuleBootFailure is continue', async () => { + const init = new BackendInitializer([ + ...baseFactories, + mockServices.rootConfig.factory({ + data: { + backend: { + startup: { default: { onPluginModuleBootFailure: 'continue' } }, + }, + }, + }), + ]); + init.add(testPlugin); + init.add( + createBackendModule({ + pluginId: 'test', + moduleId: 'mod', + register(reg) { + reg.registerInit({ + deps: {}, + async init() { + throw new Error('NOPE'); + }, + }); + }, + }), + ); + await expect(init.start()).resolves.not.toThrow(); + }); + + it('should forward errors for plugin modules explicitly marked to abort when the default is continue', async () => { + const init = new BackendInitializer([ + ...baseFactories, + mockServices.rootConfig.factory({ + data: { + backend: { + startup: { + default: { onPluginModuleBootFailure: 'continue' }, + plugins: { + test: { + modules: { mod: { onPluginModuleBootFailure: 'abort' } }, + }, + }, + }, + }, + }, + }), + ]); + init.add(testPlugin); + init.add( + createBackendModule({ + pluginId: 'test', + moduleId: 'mod', + register(reg) { + reg.registerInit({ + deps: {}, + async init() { + throw new Error('NOPE'); + }, + }); + }, + }), + ); + await expect(init.start()).rejects.toThrow( + "Module 'mod' for plugin 'test' startup failed; caused by Error: NOPE", + ); + }); + it('should forward errors when multiple plugins fail to start', async () => { const init = new BackendInitializer([]); init.add( diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.ts b/packages/backend-app-api/src/wiring/BackendInitializer.ts index f521966d4d..a666f53f29 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.ts @@ -365,17 +365,38 @@ export class BackendInitializer { } await tree.parallelTopologicalTraversal( async ({ moduleId, moduleInit }) => { - const moduleDeps = await this.#getInitDeps( - moduleInit.init.deps, - pluginId, - moduleId, - ); - await moduleInit.init.func(moduleDeps).catch(error => { - throw new ForwardedError( - `Module '${moduleId}' for plugin '${pluginId}' startup failed`, - error, + const isModuleBootFailurePermitted = + this.#getPluginModuleBootFailurePredicate( + pluginId, + moduleId, + rootConfig, ); - }); + + try { + const moduleDeps = await this.#getInitDeps( + moduleInit.init.deps, + pluginId, + moduleId, + ); + await moduleInit.init.func(moduleDeps).catch(error => { + throw new ForwardedError( + `Module '${moduleId}' for plugin '${pluginId}' startup failed`, + error, + ); + }); + } catch (error: unknown) { + assertError(error); + if (isModuleBootFailurePermitted) { + initLogger.onPermittedPluginModuleFailure( + pluginId, + moduleId, + error, + ); + } else { + initLogger.onPluginModuleFailed(pluginId, moduleId, error); + throw error; + } + } }, ); } @@ -649,6 +670,24 @@ export class BackendInitializer { return pluginStartupBootFailureValue === 'continue'; } + + #getPluginModuleBootFailurePredicate( + pluginId: string, + moduleId: string, + config?: Config, + ): boolean { + const defaultStartupBootFailureValue = + config?.getOptionalString( + 'backend.startup.default.onPluginModuleBootFailure', + ) ?? 'abort'; + + const pluginModuleStartupBootFailureValue = + config?.getOptionalString( + `backend.startup.plugins.${pluginId}.modules.${moduleId}.onPluginModuleBootFailure`, + ) ?? defaultStartupBootFailureValue; + + return pluginModuleStartupBootFailureValue === 'continue'; + } } function toInternalBackendFeature( diff --git a/packages/backend-app-api/src/wiring/createInitializationLogger.ts b/packages/backend-app-api/src/wiring/createInitializationLogger.ts index 0978cfb0c8..600826a435 100644 --- a/packages/backend-app-api/src/wiring/createInitializationLogger.ts +++ b/packages/backend-app-api/src/wiring/createInitializationLogger.ts @@ -29,6 +29,12 @@ export function createInitializationLogger( onPluginStarted(pluginId: string): void; onPluginFailed(pluginId: string, error: Error): void; onPermittedPluginFailure(pluginId: string, error: Error): void; + onPluginModuleFailed(pluginId: string, moduleId: string, error: Error): void; + onPermittedPluginModuleFailure( + pluginId: string, + moduleId: string, + error: Error, + ): void; onAllStarted(): void; } { const logger = rootLogger?.child({ type: 'initialization' }); @@ -87,6 +93,26 @@ export function createInitializationLogger( error, ); }, + onPluginModuleFailed(pluginId: string, moduleId: string, error: Error) { + const status = + starting.size > 0 + ? `, waiting for ${starting.size} other plugins to finish before shutting down the process` + : ''; + logger?.error( + `Module ${moduleId} in Plugin '${pluginId}' threw an error during startup${status}.`, + error, + ); + }, + onPermittedPluginModuleFailure( + pluginId: string, + moduleId: string, + error: Error, + ) { + logger?.error( + `Module ${moduleId} in Plugin '${pluginId}' threw an error during startup, but boot failure is permitted for this plugin module so startup will continue.`, + error, + ); + }, onAllStarted() { logger?.info(`Plugin initialization complete${getInitStatus()}`);