diff --git a/.changeset/gorgeous-dragons-peel.md b/.changeset/gorgeous-dragons-peel.md new file mode 100644 index 0000000000..d067ca0ae7 --- /dev/null +++ b/.changeset/gorgeous-dragons-peel.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-plugin-api': patch +'@backstage/backend-app-api': patch +--- + +Allow modules to register extension points. diff --git a/.changeset/perfect-trainers-invite.md b/.changeset/perfect-trainers-invite.md new file mode 100644 index 0000000000..6b543005cd --- /dev/null +++ b/.changeset/perfect-trainers-invite.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-incremental-ingestion': patch +--- + +Export new alpha `incrementalIngestionProvidersExtensionPoint` for registering incremental providers, rather than the providers being passed as options to the backend module. diff --git a/packages/backend-app-api/src/lib/DependencyGraph.test.ts b/packages/backend-app-api/src/lib/DependencyGraph.test.ts new file mode 100644 index 0000000000..65efac0252 --- /dev/null +++ b/packages/backend-app-api/src/lib/DependencyGraph.test.ts @@ -0,0 +1,245 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DependencyGraph } from './DependencyGraph'; + +describe('DependencyGraph', () => { + it('should be empty', async () => { + const empty = DependencyGraph.fromMap({}); + expect(empty.findUnsatisfiedDeps()).toEqual([]); + expect(empty.detectCircularDependency()).toBeUndefined(); + await expect( + empty.parallelTopologicalTraversal(async id => id), + ).resolves.toEqual([]); + }); + + describe('detectCircularDependency', () => { + it('should return undefined with not deps', () => { + expect( + DependencyGraph.fromMap({ + 1: {}, + 2: {}, + 3: {}, + 4: {}, + }).detectCircularDependency(), + ).toBeUndefined(); + }); + + it('should return undefined with no circular deps', async () => { + expect( + DependencyGraph.fromMap({ + 1: { provides: ['a'] }, + 2: { consumes: ['a'], provides: ['b', 'c'] }, + 3: { consumes: ['b'] }, + 4: { consumes: ['c'] }, + }).detectCircularDependency(), + ).toBeUndefined(); + }); + + it('should detect an immediate circular dep', async () => { + expect( + DependencyGraph.fromMap({ + 1: { provides: ['a'], consumes: ['a'] }, + }).detectCircularDependency(), + ).toEqual(['1', '1']); + }); + + it('should detect a small circular dep', async () => { + expect( + DependencyGraph.fromMap({ + 1: { provides: ['a'], consumes: ['b'] }, + 2: { provides: ['b'], consumes: ['a'] }, + }).detectCircularDependency(), + ).toEqual(['1', '2', '1']); + }); + + it('should detect a larger distant circular dep', async () => { + expect( + DependencyGraph.fromMap({ + 1: { provides: ['a'] }, + 2: { provides: ['b'], consumes: ['a', 'e'] }, + 3: { provides: ['c'], consumes: ['b'] }, + 4: { provides: ['d', 'e'], consumes: ['c', 'a'] }, + }).detectCircularDependency(), + ).toEqual(['2', '3', '4', '2']); + }); + }); + + describe('findUnsatisfiedDeps', () => { + it('should return nothing with no deps', () => { + expect( + DependencyGraph.fromMap({ + 1: {}, + 2: {}, + 3: {}, + 4: {}, + }).findUnsatisfiedDeps(), + ).toEqual([]); + }); + + it('should return nothing when all deps are satisfied', async () => { + expect( + DependencyGraph.fromMap({ + 1: { provides: ['a'] }, + 2: { consumes: ['a'], provides: ['b', 'c'] }, + 3: { consumes: ['b'] }, + 4: { consumes: ['c'] }, + }).findUnsatisfiedDeps(), + ).toEqual([]); + }); + + it('should find a single unsatisfied dep', async () => { + expect( + DependencyGraph.fromMap({ + 1: { consumes: ['a'] }, + }).findUnsatisfiedDeps(), + ).toEqual([{ value: '1', unsatisfied: ['a'] }]); + }); + + it('should handle circular dependencies', async () => { + expect( + DependencyGraph.fromMap({ + 1: { consumes: ['a'], provides: ['a'] }, + }).findUnsatisfiedDeps(), + ).toEqual([]); + + expect( + DependencyGraph.fromMap({ + 1: { consumes: ['a'], provides: ['b'] }, + 2: { consumes: ['b'], provides: ['a'] }, + }).findUnsatisfiedDeps(), + ).toEqual([]); + + expect( + DependencyGraph.fromMap({ + 1: { consumes: ['a'] }, + 2: { consumes: ['b'], provides: ['c'] }, + 3: { consumes: ['c'], provides: ['a', 'b'] }, + }).findUnsatisfiedDeps(), + ).toEqual([]); + }); + + it('should find multiple unsatisfied deps for one node', async () => { + expect( + DependencyGraph.fromMap({ + 1: { provides: ['a'], consumes: ['b'] }, + 2: { provides: ['b'], consumes: ['a', 'd', 'e'] }, + }).findUnsatisfiedDeps(), + ).toEqual([{ value: '2', unsatisfied: ['d', 'e'] }]); + }); + + it('should find multiple unsatisfied deps for multiple nodes', async () => { + expect( + DependencyGraph.fromMap({ + 1: { provides: ['a'] }, + 2: { provides: ['b'], consumes: ['a', 'd', 'e'] }, + 3: { provides: [], consumes: ['b'] }, + 4: { provides: [], consumes: ['c', 'a'] }, + }).findUnsatisfiedDeps(), + ).toEqual([ + { value: '2', unsatisfied: ['d', 'e'] }, + { value: '4', unsatisfied: ['c'] }, + ]); + }); + }); + + describe('parallelTopologicalTraversal', () => { + it('should traverse with no deps', async () => { + await expect( + DependencyGraph.fromMap({ + 1: {}, + 2: {}, + 3: {}, + 4: {}, + }).parallelTopologicalTraversal(async id => id), + ).resolves.toEqual(['1', '2', '3', '4']); + }); + + it('should traverse with a few deps', async () => { + await expect( + DependencyGraph.fromMap({ + 1: { provides: ['a'] }, + 2: { consumes: ['a'], provides: ['b', 'c'] }, + 3: { consumes: ['b'] }, + 4: { consumes: ['c'] }, + }).parallelTopologicalTraversal(async id => id), + ).resolves.toEqual(['1', '2', '3', '4']); + }); + + it('should traverse in reverse', async () => { + await expect( + DependencyGraph.fromMap({ + 1: { consumes: ['c'] }, + 2: { provides: ['c'], consumes: ['b'] }, + 3: { provides: ['b'], consumes: ['a'] }, + 4: { provides: ['a'] }, + }).parallelTopologicalTraversal(async id => id), + ).resolves.toEqual(['4', '3', '2', '1']); + }); + + it('should execute in parallel', async () => { + await expect( + DependencyGraph.fromMap({ + 1: { provides: ['a'] }, + 2: { provides: ['b'], consumes: ['a'] }, + 3: { provides: ['c'], consumes: ['a'] }, + 4: { consumes: ['b'] }, + 5: { consumes: ['c'] }, + }).parallelTopologicalTraversal(async id => id), + ).resolves.toEqual(['1', '2', '3', '4', '5']); + + // Same as above, but with 2 being delayed + await expect( + DependencyGraph.fromMap({ + 1: { provides: ['a'] }, + 2: { provides: ['b'], consumes: ['a'] }, + 3: { provides: ['c'], consumes: ['a'] }, + 4: { consumes: ['b'] }, + 5: { consumes: ['c'] }, + }).parallelTopologicalTraversal(async id => { + // When delaying 2 we expect 3 and 5 to complete before 2 and 4 + if (id === '2') { + await new Promise(resolve => setTimeout(resolve, 100)); + } + return id; + }), + ).resolves.toEqual(['1', '3', '5', '2', '4']); + }); + + it('should detect circular dependencies', async () => { + await expect( + DependencyGraph.fromMap({ + 1: { provides: ['a'], consumes: ['a'] }, + }).parallelTopologicalTraversal(async id => id), + ).rejects.toThrow('Circular dependency detected'); + + await expect( + DependencyGraph.fromMap({ + 1: { provides: ['a'], consumes: ['b'] }, + 2: { provides: ['b'], consumes: ['a'] }, + }).parallelTopologicalTraversal(async id => id), + ).rejects.toThrow('Circular dependency detected'); + + await expect( + DependencyGraph.fromMap({ + 1: { provides: ['a'] }, + 2: { provides: ['c'], consumes: ['a', 'b'] }, + 3: { provides: ['b'], consumes: ['a', 'c'] }, + }).parallelTopologicalTraversal(async id => id), + ).rejects.toThrow('Circular dependency detected'); + }); + }); +}); diff --git a/packages/backend-app-api/src/lib/DependencyGraph.ts b/packages/backend-app-api/src/lib/DependencyGraph.ts new file mode 100644 index 0000000000..8fd679f555 --- /dev/null +++ b/packages/backend-app-api/src/lib/DependencyGraph.ts @@ -0,0 +1,197 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +interface NodeInput { + value: T; + consumes?: Iterable; + provides?: Iterable; +} + +/** @internal */ +class Node { + static from(input: NodeInput) { + return new Node( + input.value, + input.consumes ? new Set(input.consumes) : new Set(), + input.provides ? new Set(input.provides) : new Set(), + ); + } + + private constructor( + readonly value: T, + readonly consumes: Set, + readonly provides: Set, + ) {} +} + +/** + * Internal helper to help validate and traverse a dependency graph. + * @internal + */ +export class DependencyGraph { + static fromMap( + nodes: Record, 'value'>>, + ): DependencyGraph { + return this.fromIterable( + Object.entries(nodes).map(([key, node]) => ({ + value: String(key), + ...node, + })), + ); + } + + static fromIterable( + nodeInputs: Iterable>, + ): DependencyGraph { + const nodes = new Array>(); + for (const nodeInput of nodeInputs) { + nodes.push(Node.from(nodeInput)); + } + + return new DependencyGraph(nodes); + } + + #nodes: Array>; + #allProvided: Set; + + private constructor(nodes: Array>) { + this.#nodes = nodes; + this.#allProvided = new Set(); + + for (const node of this.#nodes.values()) { + for (const produced of node.provides) { + this.#allProvided.add(produced); + } + } + } + + // Find all nodes that consume dependencies that are not provided by any other node + findUnsatisfiedDeps(): Array<{ value: T; unsatisfied: string[] }> { + const unsatisfiedDependencies = []; + for (const node of this.#nodes.values()) { + const unsatisfied = Array.from(node.consumes).filter( + id => !this.#allProvided.has(id), + ); + if (unsatisfied.length > 0) { + unsatisfiedDependencies.push({ value: node.value, unsatisfied }); + } + } + return unsatisfiedDependencies; + } + + // Detect circular dependencies within the graph, returning the path of nodes that + // form a cycle, with the same node as the first and last element of the array. + detectCircularDependency(): T[] | undefined { + for (const startNode of this.#nodes) { + const visited = new Set>(); + const stack = new Array<[node: Node, path: T[]]>([ + startNode, + [startNode.value], + ]); + + while (stack.length > 0) { + const [node, path] = stack.pop()!; + if (visited.has(node)) { + continue; + } + visited.add(node); + for (const produced of node.provides) { + const consumerNodes = this.#nodes.filter(other => + other.consumes.has(produced), + ); + for (const consumer of consumerNodes) { + if (consumer === startNode) { + return [...path, startNode.value]; + } + if (!visited.has(consumer)) { + stack.push([consumer, [...path, consumer.value]]); + } + } + } + } + } + return undefined; + } + + /** + * Traverses the dependency graph in topological order, calling the provided + * function for each node and waiting for it to resolve. + * + * The nodes are traversed in parallel, but in such a way that no node is + * visited before all of its dependencies. + * + * Dependencies of nodes that are not produced by any other nodes will be ignored. + */ + async parallelTopologicalTraversal( + fn: (value: T) => Promise, + ): Promise { + const allProvided = this.#allProvided; + const producedSoFar = new Set(); + const waiting = new Set(this.#nodes.values()); + const visited = new Set>(); + const results = new Array(); + let inFlight = 0; // Keep track of how many callbacks are in flight, so that we know if we got stuck + + // Find all nodes that have no dependencies that have not already been produced by visited nodes + async function processMoreNodes() { + if (waiting.size === 0) { + return; + } + const nodesToProcess = []; + for (const node of waiting) { + let ready = true; + for (const consumed of node.consumes) { + if (allProvided.has(consumed) && !producedSoFar.has(consumed)) { + ready = false; + continue; + } + } + if (ready) { + nodesToProcess.push(node); + } + } + + for (const node of nodesToProcess) { + waiting.delete(node); + } + + if (nodesToProcess.length === 0 && inFlight === 0) { + // We expect the caller to check for circular dependencies before + // traversal, so this error should never happen + throw new Error('Circular dependency detected'); + } + + await Promise.all(nodesToProcess.map(processNode)); + } + + // Process an individual node, and then add its produced dependencies to the set of available products + async function processNode(node: Node) { + visited.add(node); + inFlight += 1; + + const result = await fn(node.value); + results.push(result); + + node.provides.forEach(produced => producedSoFar.add(produced)); + inFlight -= 1; + await processMoreNodes(); + } + + await processMoreNodes(); + + return results; + } +} diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.test.ts b/packages/backend-app-api/src/wiring/BackendInitializer.test.ts index b05194f594..ba637b7b5b 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.test.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.test.ts @@ -20,10 +20,15 @@ import { coreServices, createBackendPlugin, createBackendModule, + createExtensionPoint, } from '@backstage/backend-plugin-api'; import { BackendInitializer } from './BackendInitializer'; import { ServiceRegistry } from './ServiceRegistry'; -import { rootLifecycleServiceFactory } from '../services/implementations'; +import { + lifecycleServiceFactory, + loggerServiceFactory, + rootLifecycleServiceFactory, +} from '../services/implementations'; const rootRef = createServiceRef<{ x: number }>({ id: '1', @@ -44,6 +49,17 @@ class MockLogger { } } +const baseFactories = [ + lifecycleServiceFactory(), + rootLifecycleServiceFactory(), + createServiceFactory({ + service: coreServices.rootLogger, + deps: {}, + factory: () => new MockLogger(), + })(), + loggerServiceFactory(), +]; + describe('BackendInitializer', () => { it('should initialize root scoped services', async () => { const rootFactory = jest.fn(); @@ -75,6 +91,65 @@ describe('BackendInitializer', () => { expect(pluginFactory).not.toHaveBeenCalled(); }); + it('should initialize modules with extension points', async () => { + expect.assertions(3); + + const extensionPoint = createExtensionPoint<{ values: string[] }>({ + id: 'a', + }); + const init = new BackendInitializer(new ServiceRegistry(baseFactories)); + + init.add( + createBackendModule({ + pluginId: 'test', + moduleId: 'modA', + register(reg) { + reg.registerInit({ + deps: { extension: extensionPoint }, + async init({ extension }) { + expect(extension.values).toEqual(['b']); + extension.values.push('a'); + }, + }); + }, + })(), + ); + + init.add( + createBackendModule({ + pluginId: 'test', + moduleId: 'modB', + register(reg) { + const values = ['b']; + reg.registerExtensionPoint(extensionPoint, { values }); + reg.registerInit({ + deps: {}, + async init() { + expect(values).toEqual(['b', 'a', 'c']); + }, + }); + }, + })(), + ); + + init.add( + createBackendModule({ + pluginId: 'test', + moduleId: 'modC', + register(reg) { + reg.registerInit({ + deps: { extension: extensionPoint }, + async init({ extension }) { + expect(extension.values).toEqual(['b', 'a']); + extension.values.push('c'); + }, + }); + }, + })(), + ); + await init.start(); + }); + it('should forward errors when plugins fail to start', async () => { const init = new BackendInitializer(new ServiceRegistry([])); init.add( @@ -175,4 +250,48 @@ describe('BackendInitializer', () => { "Module 'mod' for plugin 'test' is already registered", ); }); + + it('should reject modules with circular dependencies', async () => { + const extA = createExtensionPoint({ id: 'a' }); + const extB = createExtensionPoint({ id: 'b' }); + const init = new BackendInitializer( + new ServiceRegistry([ + rootLifecycleServiceFactory(), + createServiceFactory({ + service: coreServices.rootLogger, + deps: {}, + factory: () => new MockLogger(), + })(), + ]), + ); + init.add( + createBackendModule({ + pluginId: 'test', + moduleId: 'modA', + register(reg) { + reg.registerExtensionPoint(extA, 'a'); + reg.registerInit({ + deps: { ext: extB }, + async init() {}, + }); + }, + })(), + ); + init.add( + createBackendModule({ + pluginId: 'test', + moduleId: 'modB', + register(reg) { + reg.registerExtensionPoint(extB, 'b'); + reg.registerInit({ + deps: { ext: extA }, + async init() {}, + }); + }, + })(), + ); + await expect(init.start()).rejects.toThrow( + "Circular dependency detected for modules of plugin 'test', 'modA' -> 'modB' -> 'modA'", + ); + }); }); diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.ts b/packages/backend-app-api/src/wiring/BackendInitializer.ts index 6bc4e995e2..06201dfb2b 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.ts @@ -26,8 +26,9 @@ import { EnumerableServiceHolder, ServiceOrExtensionPoint } from './types'; // Direct internal import to avoid duplication // eslint-disable-next-line @backstage/no-forbidden-package-imports import { InternalBackendFeature } from '@backstage/backend-plugin-api/src/wiring/types'; -import { ForwardedError } from '@backstage/errors'; +import { ForwardedError, ConflictError } from '@backstage/errors'; import { featureDiscoveryServiceRef } from '@backstage/backend-plugin-api/alpha'; +import { DependencyGraph } from '../lib/DependencyGraph'; export interface BackendRegisterInit { consumes: Set; @@ -161,7 +162,7 @@ export class BackendInitializer { for (const r of feature.getRegistrations()) { const provides = new Set>(); - if (r.type === 'plugin') { + if (r.type === 'plugin' || r.type === 'module') { for (const [extRef, extImpl] of r.extensionPoints) { if (this.#extensionPoints.has(extRef)) { throw new Error( @@ -210,21 +211,41 @@ export class BackendInitializer { await Promise.all( allPluginIds.map(async pluginId => { // Modules are initialized before plugins, so that they can provide extension to the plugin - const modules = moduleInits.get(pluginId) ?? []; - await Promise.all( - Array.from(modules).map(async ([moduleId, moduleInit]) => { - const moduleDeps = await this.#getInitDeps( - moduleInit.init.deps, - pluginId, + const modules = moduleInits.get(pluginId); + if (modules) { + const tree = DependencyGraph.fromIterable( + Array.from(modules).map(([moduleId, moduleInit]) => ({ + value: { moduleId, moduleInit }, + // Relationships are reversed at this point since we're only interested in the extension points. + // If a modules provides extension point A we want it to be initialized AFTER all modules + // that depend on extension point A, so that they can provide their extensions. + consumes: Array.from(moduleInit.provides).map(p => p.id), + provides: Array.from(moduleInit.consumes).map(c => c.id), + })), + ); + const circular = tree.detectCircularDependency(); + if (circular) { + throw new ConflictError( + `Circular dependency detected for modules of plugin '${pluginId}', ${circular + .map(({ moduleId }) => `'${moduleId}'`) + .join(' -> ')}`, ); - await moduleInit.init.func(moduleDeps).catch(error => { - throw new ForwardedError( - `Module '${moduleId}' for plugin '${pluginId}' startup failed`, - error, + } + await tree.parallelTopologicalTraversal( + async ({ moduleId, moduleInit }) => { + const moduleDeps = await this.#getInitDeps( + moduleInit.init.deps, + pluginId, ); - }); - }), - ); + await moduleInit.init.func(moduleDeps).catch(error => { + throw new ForwardedError( + `Module '${moduleId}' for plugin '${pluginId}' startup failed`, + error, + ); + }); + }, + ); + } // Once all modules have been initialized, we can initialize the plugin itself const pluginInit = pluginInits.get(pluginId); diff --git a/packages/backend-plugin-api/api-report.md b/packages/backend-plugin-api/api-report.md index 6dbbee1936..9fa2149e43 100644 --- a/packages/backend-plugin-api/api-report.md +++ b/packages/backend-plugin-api/api-report.md @@ -31,6 +31,11 @@ export interface BackendModuleConfig { // @public export interface BackendModuleRegistrationPoints { + // (undocumented) + registerExtensionPoint( + ref: ExtensionPoint, + impl: TExtensionPoint, + ): void; // (undocumented) registerInit< Deps extends { diff --git a/packages/backend-plugin-api/src/wiring/factories.test.ts b/packages/backend-plugin-api/src/wiring/factories.test.ts index ae199d84e4..17b0eed55f 100644 --- a/packages/backend-plugin-api/src/wiring/factories.test.ts +++ b/packages/backend-plugin-api/src/wiring/factories.test.ts @@ -149,6 +149,7 @@ describe('createBackendModule', () => { type: 'module', pluginId: 'x', moduleId: 'y', + extensionPoints: [], init: { deps: expect.any(Object), func: expect.any(Function), diff --git a/packages/backend-plugin-api/src/wiring/factories.ts b/packages/backend-plugin-api/src/wiring/factories.ts index ecaccdc6de..c2ff1f84b9 100644 --- a/packages/backend-plugin-api/src/wiring/factories.ts +++ b/packages/backend-plugin-api/src/wiring/factories.ts @@ -159,13 +159,14 @@ export function createBackendPlugin( */ export interface BackendModuleConfig { /** - * The ID of this plugin. + * Should exactly match the `id` of the plugin that the module extends. * * @see {@link https://backstage.io/docs/backend-system/architecture/naming-patterns | Recommended naming patterns} */ pluginId: string; + /** - * Should exactly match the `id` of the plugin that the module extends. + * The ID of this module, used to identify the module and ensure that it is not installed twice. */ moduleId: string; register(reg: BackendModuleRegistrationPoints): void; @@ -194,10 +195,20 @@ export function createBackendModule( if (registrations) { return registrations; } + const extensionPoints: InternalBackendPluginRegistration['extensionPoints'] = + []; let init: InternalBackendModuleRegistration['init'] | undefined = undefined; c.register({ + registerExtensionPoint(ext, impl) { + if (init) { + throw new Error( + 'registerExtensionPoint called after registerInit', + ); + } + extensionPoints.push([ext, impl]); + }, registerInit(regInit) { if (init) { throw new Error('registerInit must only be called once'); @@ -220,6 +231,7 @@ export function createBackendModule( type: 'module', pluginId: c.pluginId, moduleId: c.moduleId, + extensionPoints, init, }, ]; diff --git a/packages/backend-plugin-api/src/wiring/types.ts b/packages/backend-plugin-api/src/wiring/types.ts index 028c2b87da..e2ed6ce387 100644 --- a/packages/backend-plugin-api/src/wiring/types.ts +++ b/packages/backend-plugin-api/src/wiring/types.ts @@ -59,6 +59,10 @@ export interface BackendPluginRegistrationPoints { * @public */ export interface BackendModuleRegistrationPoints { + registerExtensionPoint( + ref: ExtensionPoint, + impl: TExtensionPoint, + ): void; registerInit(options: { deps: { [name in keyof Deps]: ServiceRef | ExtensionPoint; @@ -105,6 +109,7 @@ export interface InternalBackendModuleRegistration { pluginId: string; moduleId: string; type: 'module'; + extensionPoints: Array, unknown]>; init: { deps: Record | ExtensionPoint>; func(deps: Record): Promise; diff --git a/plugins/catalog-backend-module-incremental-ingestion/alpha-api-report.md b/plugins/catalog-backend-module-incremental-ingestion/alpha-api-report.md index b11ec43cf5..995c6e4893 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/alpha-api-report.md +++ b/plugins/catalog-backend-module-incremental-ingestion/alpha-api-report.md @@ -4,16 +4,23 @@ ```ts import { BackendFeature } from '@backstage/backend-plugin-api'; +import { ExtensionPoint } from '@backstage/backend-plugin-api'; import { IncrementalEntityProvider } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; import { IncrementalEntityProviderOptions } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; // @alpha -export const catalogModuleIncrementalIngestionEntityProvider: (options: { - providers: { - provider: IncrementalEntityProvider; +export const catalogModuleIncrementalIngestionEntityProvider: () => BackendFeature; + +// @alpha +export interface IncrementalIngestionProviderExtensionPoint { + addProvider(config: { options: IncrementalEntityProviderOptions; - }[]; -}) => BackendFeature; + provider: IncrementalEntityProvider; + }): void; +} + +// @alpha +export const incrementalIngestionProvidersExtensionPoint: ExtensionPoint; // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.test.ts index ae480048d1..7c3dfdf226 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.test.ts @@ -15,12 +15,18 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { coreServices } from '@backstage/backend-plugin-api'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; import { startTestBackend } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; import { IncrementalEntityProvider } from '../types'; -import { catalogModuleIncrementalIngestionEntityProvider } from './catalogModuleIncrementalIngestionEntityProvider'; +import { + catalogModuleIncrementalIngestionEntityProvider, + incrementalIngestionProvidersExtensionPoint, +} from './catalogModuleIncrementalIngestionEntityProvider'; describe('catalogModuleIncrementalIngestionEntityProvider', () => { it('should register provider at the catalog extension point', async () => { @@ -57,18 +63,26 @@ describe('catalogModuleIncrementalIngestionEntityProvider', () => { [coreServices.scheduler, scheduler], ], features: [ - catalogModuleIncrementalIngestionEntityProvider({ - providers: [ - { - provider: provider1, - options: { - burstInterval: { seconds: 1 }, - burstLength: { seconds: 1 }, - restLength: { seconds: 1 }, + catalogModuleIncrementalIngestionEntityProvider(), + createBackendModule({ + pluginId: 'catalog', + moduleId: 'incrementalTest', + register(env) { + env.registerInit({ + deps: { extension: incrementalIngestionProvidersExtensionPoint }, + async init({ extension }) { + extension.addProvider({ + provider: provider1, + options: { + burstInterval: { seconds: 1 }, + burstLength: { seconds: 1 }, + restLength: { seconds: 1 }, + }, + }); }, - }, - ], - }), + }); + }, + })(), ], }); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.ts index 78d28d7b07..4e94088ef6 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.ts @@ -17,6 +17,7 @@ import { coreServices, createBackendModule, + createExtensionPoint, } from '@backstage/backend-plugin-api'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; import { @@ -25,56 +26,111 @@ import { } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; import { WrapperProviders } from './WrapperProviders'; +/** + * @alpha + * Interface for {@link incrementalIngestionProvidersExtensionPoint}. + */ +export interface IncrementalIngestionProviderExtensionPoint { + /** Adds a new incremental entity provider */ + addProvider(config: { + options: IncrementalEntityProviderOptions; + provider: IncrementalEntityProvider; + }): void; +} + +/** + * @alpha + * + * Extension point for registering incremental ingestion providers. + * The `catalogModuleIncrementalIngestionEntityProvider` must be installed for these providers to work. + * + * @example + * + * ```ts + * backend.add(createBackendModule({ + * pluginId: 'catalog', + * moduleId: 'myIncrementalProvider', + * register(env) { + * env.registerInit({ + * deps: { + * extension: incrementalIngestionProvidersExtensionPoint, + * }, + * async init({ extension }) { + * extension.addProvider({ + * burstInterval: ..., + * burstLength: ..., + * restLength: ..., + * }, { + * next(context, cursor) { + * ... + * }, + * ... + * }) + * }) + * }) + * } + * })) + * ``` + */ +export const incrementalIngestionProvidersExtensionPoint = + createExtensionPoint({ + id: 'catalog.incrementalIngestionProvider.providers', + }); + /** * Registers the incremental entity provider with the catalog processing extension point. * * @alpha */ export const catalogModuleIncrementalIngestionEntityProvider = - createBackendModule( - (options: { - providers: Array<{ + createBackendModule({ + pluginId: 'catalog', + moduleId: 'incrementalIngestionEntityProvider', + register(env) { + const addedProviders = new Array<{ provider: IncrementalEntityProvider; options: IncrementalEntityProviderOptions; - }>; - }) => ({ - pluginId: 'catalog', - moduleId: 'incrementalIngestionEntityProvider', - register(env) { - env.registerInit({ - deps: { - catalog: catalogProcessingExtensionPoint, - config: coreServices.rootConfig, - database: coreServices.database, - httpRouter: coreServices.httpRouter, - logger: coreServices.logger, - scheduler: coreServices.scheduler, - }, - async init({ - catalog, + }>(); + + env.registerExtensionPoint(incrementalIngestionProvidersExtensionPoint, { + addProvider({ options, provider }) { + addedProviders.push({ options, provider }); + }, + }); + + env.registerInit({ + deps: { + catalog: catalogProcessingExtensionPoint, + config: coreServices.rootConfig, + database: coreServices.database, + httpRouter: coreServices.httpRouter, + logger: coreServices.logger, + scheduler: coreServices.scheduler, + }, + async init({ + catalog, + config, + database, + httpRouter, + logger, + scheduler, + }) { + const client = await database.getClient(); + + const providers = new WrapperProviders({ config, - database, - httpRouter, logger, + client, scheduler, - }) { - const client = await database.getClient(); + }); - const providers = new WrapperProviders({ - config, - logger, - client, - scheduler, - }); + for (const entry of addedProviders) { + const wrapped = providers.wrap(entry.provider, entry.options); + catalog.addEntityProvider(wrapped); + } - for (const entry of options.providers) { - const wrapped = providers.wrap(entry.provider, entry.options); - catalog.addEntityProvider(wrapped); - } - - httpRouter.use(await providers.adminRouter()); - }, - }); - }, - }), - ); + httpRouter.use(await providers.adminRouter()); + }, + }); + }, + }); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts index 9fcee99e01..37ecf60f7c 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts @@ -14,4 +14,8 @@ * limitations under the License. */ -export { catalogModuleIncrementalIngestionEntityProvider } from './catalogModuleIncrementalIngestionEntityProvider'; +export { + catalogModuleIncrementalIngestionEntityProvider, + incrementalIngestionProvidersExtensionPoint, + type IncrementalIngestionProviderExtensionPoint, +} from './catalogModuleIncrementalIngestionEntityProvider'; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts index e7e7a900b9..be1a6dcfc0 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts @@ -20,12 +20,16 @@ import { createBackend } from '@backstage/backend-defaults'; import { coreServices, + createBackendModule, createServiceFactory, } from '@backstage/backend-plugin-api'; import { ConfigReader } from '@backstage/config'; import { catalogPlugin } from '@backstage/plugin-catalog-backend/alpha'; import { IncrementalEntityProvider } from '.'; -import { catalogModuleIncrementalIngestionEntityProvider } from './alpha'; +import { + catalogModuleIncrementalIngestionEntityProvider, + incrementalIngestionProvidersExtensionPoint, +} from './alpha'; const provider: IncrementalEntityProvider = { getProviderName: () => 'test-provider', @@ -62,19 +66,27 @@ async function main() { }); backend.add(catalogPlugin()); + backend.add(catalogModuleIncrementalIngestionEntityProvider()); backend.add( - catalogModuleIncrementalIngestionEntityProvider({ - providers: [ - { - provider: provider, - options: { - burstInterval: { seconds: 1 }, - burstLength: { seconds: 10 }, - restLength: { seconds: 10 }, + createBackendModule({ + pluginId: 'catalog', + moduleId: 'incrementalTestProvider', + register(reg) { + reg.registerInit({ + deps: { extension: incrementalIngestionProvidersExtensionPoint }, + async init({ extension }) { + extension.addProvider({ + provider: provider, + options: { + burstInterval: { seconds: 1 }, + burstLength: { seconds: 10 }, + restLength: { seconds: 10 }, + }, + }); }, - }, - ], - }), + }); + }, + })(), ); await backend.start();