From e2b6396a127488342d9b63743e40e29352e94d92 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 10 Aug 2023 19:11:39 +0200 Subject: [PATCH] catalog-backend-module-incremental-ingestion: move providers to extension point instead of options Signed-off-by: Patrik Oldsberg --- .changeset/perfect-trainers-invite.md | 5 + .../alpha-api-report.md | 17 ++- ...IncrementalIngestionEntityProvider.test.ts | 40 ++++-- ...oduleIncrementalIngestionEntityProvider.ts | 135 ++++++++++++------ .../src/module/index.ts | 6 +- .../src/run.ts | 36 +++-- 6 files changed, 168 insertions(+), 71 deletions(-) create mode 100644 .changeset/perfect-trainers-invite.md diff --git a/.changeset/perfect-trainers-invite.md b/.changeset/perfect-trainers-invite.md new file mode 100644 index 0000000000..6b543005cd --- /dev/null +++ b/.changeset/perfect-trainers-invite.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-incremental-ingestion': patch +--- + +Export new alpha `incrementalIngestionProvidersExtensionPoint` for registering incremental providers, rather than the providers being passed as options to the backend module. diff --git a/plugins/catalog-backend-module-incremental-ingestion/alpha-api-report.md b/plugins/catalog-backend-module-incremental-ingestion/alpha-api-report.md index b11ec43cf5..995c6e4893 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/alpha-api-report.md +++ b/plugins/catalog-backend-module-incremental-ingestion/alpha-api-report.md @@ -4,16 +4,23 @@ ```ts import { BackendFeature } from '@backstage/backend-plugin-api'; +import { ExtensionPoint } from '@backstage/backend-plugin-api'; import { IncrementalEntityProvider } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; import { IncrementalEntityProviderOptions } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; // @alpha -export const catalogModuleIncrementalIngestionEntityProvider: (options: { - providers: { - provider: IncrementalEntityProvider; +export const catalogModuleIncrementalIngestionEntityProvider: () => BackendFeature; + +// @alpha +export interface IncrementalIngestionProviderExtensionPoint { + addProvider(config: { options: IncrementalEntityProviderOptions; - }[]; -}) => BackendFeature; + provider: IncrementalEntityProvider; + }): void; +} + +// @alpha +export const incrementalIngestionProvidersExtensionPoint: ExtensionPoint; // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.test.ts index ae480048d1..7c3dfdf226 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.test.ts @@ -15,12 +15,18 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { coreServices } from '@backstage/backend-plugin-api'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; import { startTestBackend } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; import { IncrementalEntityProvider } from '../types'; -import { catalogModuleIncrementalIngestionEntityProvider } from './catalogModuleIncrementalIngestionEntityProvider'; +import { + catalogModuleIncrementalIngestionEntityProvider, + incrementalIngestionProvidersExtensionPoint, +} from './catalogModuleIncrementalIngestionEntityProvider'; describe('catalogModuleIncrementalIngestionEntityProvider', () => { it('should register provider at the catalog extension point', async () => { @@ -57,18 +63,26 @@ describe('catalogModuleIncrementalIngestionEntityProvider', () => { [coreServices.scheduler, scheduler], ], features: [ - catalogModuleIncrementalIngestionEntityProvider({ - providers: [ - { - provider: provider1, - options: { - burstInterval: { seconds: 1 }, - burstLength: { seconds: 1 }, - restLength: { seconds: 1 }, + catalogModuleIncrementalIngestionEntityProvider(), + createBackendModule({ + pluginId: 'catalog', + moduleId: 'incrementalTest', + register(env) { + env.registerInit({ + deps: { extension: incrementalIngestionProvidersExtensionPoint }, + async init({ extension }) { + extension.addProvider({ + provider: provider1, + options: { + burstInterval: { seconds: 1 }, + burstLength: { seconds: 1 }, + restLength: { seconds: 1 }, + }, + }); }, - }, - ], - }), + }); + }, + })(), ], }); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.ts index 78d28d7b07..81419d66ec 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/catalogModuleIncrementalIngestionEntityProvider.ts @@ -17,6 +17,7 @@ import { coreServices, createBackendModule, + createExtensionPoint, } from '@backstage/backend-plugin-api'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; import { @@ -25,56 +26,110 @@ import { } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; import { WrapperProviders } from './WrapperProviders'; +/** + * @alpha + * Interface for {@link incrementalIngestionProvidersExtensionPoint}. + */ +export interface IncrementalIngestionProviderExtensionPoint { + /** Adds a new incremental entity provider */ + addProvider(config: { + options: IncrementalEntityProviderOptions; + provider: IncrementalEntityProvider; + }): void; +} + +/** + * @alpha + * + * Extension point for registering incremental ingestion providers. + * The `catalogModuleIncrementalIngestionEntityProvider` must be installed for these providers to work. + * + * @example + * + * ```ts + * backend.add(createBackendModule({ + * pluginId: 'catalog', + * register(env) { + * env.registerInit({ + * deps: { + * extension: incrementalIngestionProvidersExtensionPoint, + * }, + * async init({ extension }) { + * extension.addProvider({ + * burstInterval: ..., + * burstLength: ..., + * restLength: ..., + * }, { + * next(context, cursor) { + * ... + * }, + * ... + * }) + * }) + * }) + * } + * })) + * ``` + */ +export const incrementalIngestionProvidersExtensionPoint = + createExtensionPoint({ + id: 'catalog.incrementalIngestionProvider.providers', + }); + /** * Registers the incremental entity provider with the catalog processing extension point. * * @alpha */ export const catalogModuleIncrementalIngestionEntityProvider = - createBackendModule( - (options: { - providers: Array<{ + createBackendModule({ + pluginId: 'catalog', + moduleId: 'incrementalIngestionEntityProvider', + register(env) { + const addedProviders = new Array<{ provider: IncrementalEntityProvider; options: IncrementalEntityProviderOptions; - }>; - }) => ({ - pluginId: 'catalog', - moduleId: 'incrementalIngestionEntityProvider', - register(env) { - env.registerInit({ - deps: { - catalog: catalogProcessingExtensionPoint, - config: coreServices.rootConfig, - database: coreServices.database, - httpRouter: coreServices.httpRouter, - logger: coreServices.logger, - scheduler: coreServices.scheduler, - }, - async init({ - catalog, + }>(); + + env.registerExtensionPoint(incrementalIngestionProvidersExtensionPoint, { + addProvider({ options, provider }) { + addedProviders.push({ options, provider }); + }, + }); + + env.registerInit({ + deps: { + catalog: catalogProcessingExtensionPoint, + config: coreServices.rootConfig, + database: coreServices.database, + httpRouter: coreServices.httpRouter, + logger: coreServices.logger, + scheduler: coreServices.scheduler, + }, + async init({ + catalog, + config, + database, + httpRouter, + logger, + scheduler, + }) { + const client = await database.getClient(); + + const providers = new WrapperProviders({ config, - database, - httpRouter, logger, + client, scheduler, - }) { - const client = await database.getClient(); + }); - const providers = new WrapperProviders({ - config, - logger, - client, - scheduler, - }); + for (const entry of addedProviders) { + const wrapped = providers.wrap(entry.provider, entry.options); + catalog.addEntityProvider(wrapped); + } - for (const entry of options.providers) { - const wrapped = providers.wrap(entry.provider, entry.options); - catalog.addEntityProvider(wrapped); - } - - httpRouter.use(await providers.adminRouter()); - }, - }); - }, - }), - ); + httpRouter.use(await providers.adminRouter()); + }, + }); + }, + }); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts index 9fcee99e01..37ecf60f7c 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts @@ -14,4 +14,8 @@ * limitations under the License. */ -export { catalogModuleIncrementalIngestionEntityProvider } from './catalogModuleIncrementalIngestionEntityProvider'; +export { + catalogModuleIncrementalIngestionEntityProvider, + incrementalIngestionProvidersExtensionPoint, + type IncrementalIngestionProviderExtensionPoint, +} from './catalogModuleIncrementalIngestionEntityProvider'; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts index e7e7a900b9..be1a6dcfc0 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts @@ -20,12 +20,16 @@ import { createBackend } from '@backstage/backend-defaults'; import { coreServices, + createBackendModule, createServiceFactory, } from '@backstage/backend-plugin-api'; import { ConfigReader } from '@backstage/config'; import { catalogPlugin } from '@backstage/plugin-catalog-backend/alpha'; import { IncrementalEntityProvider } from '.'; -import { catalogModuleIncrementalIngestionEntityProvider } from './alpha'; +import { + catalogModuleIncrementalIngestionEntityProvider, + incrementalIngestionProvidersExtensionPoint, +} from './alpha'; const provider: IncrementalEntityProvider = { getProviderName: () => 'test-provider', @@ -62,19 +66,27 @@ async function main() { }); backend.add(catalogPlugin()); + backend.add(catalogModuleIncrementalIngestionEntityProvider()); backend.add( - catalogModuleIncrementalIngestionEntityProvider({ - providers: [ - { - provider: provider, - options: { - burstInterval: { seconds: 1 }, - burstLength: { seconds: 10 }, - restLength: { seconds: 10 }, + createBackendModule({ + pluginId: 'catalog', + moduleId: 'incrementalTestProvider', + register(reg) { + reg.registerInit({ + deps: { extension: incrementalIngestionProvidersExtensionPoint }, + async init({ extension }) { + extension.addProvider({ + provider: provider, + options: { + burstInterval: { seconds: 1 }, + burstLength: { seconds: 10 }, + restLength: { seconds: 10 }, + }, + }); }, - }, - ], - }), + }); + }, + })(), ); await backend.start();