From 66d70c7d3c8450c3e2efd675c2fdba87ec1a4b3b Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 12 Jan 2023 09:38:14 -0800 Subject: [PATCH] Revert "Simplify type params where they aren't needed" This reverts commit bcb0ff4f90a3bdb9a70789ec97bbdec4ee72b311. Signed-off-by: Damon Kaswell --- .../src/engine/IncrementalIngestionEngine.ts | 8 ++++--- .../src/module/WrapperProviders.test.ts | 4 ++-- .../src/module/WrapperProviders.ts | 6 +++--- ...gestionEntityProviderCatalogModule.test.ts | 2 +- ...talIngestionEntityProviderCatalogModule.ts | 2 +- .../src/run.ts | 4 ++-- .../src/service/IncrementalCatalogBuilder.ts | 4 ++-- .../src/types.ts | 21 +++++++++++-------- 8 files changed, 28 insertions(+), 23 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts index e4d11b5f0d..6213dcf786 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts @@ -23,7 +23,7 @@ import { v4 } from 'uuid'; import { stringifyError } from '@backstage/errors'; import { EventParams, EventSubscriber } from '@backstage/plugin-events-node'; -export class IncrementalIngestionEngine +export class IncrementalIngestionEngine implements IterationEngine, EventSubscriber { private readonly restLength: Duration; @@ -32,7 +32,7 @@ export class IncrementalIngestionEngine private manager: IncrementalIngestionDatabaseManager; - constructor(private options: IterationEngineOptions) { + constructor(private options: IterationEngineOptions) { this.manager = options.manager; this.restLength = Duration.fromObject(options.restLength); this.backoff = options.backoff ?? [ @@ -347,11 +347,13 @@ export class IncrementalIngestionEngine `incremental-engine: Received ${this.providerEventTopic} event`, ); + const payload = eventPayload as TInput; + if (!provider.deltaMapper) { return; } - const update = provider.deltaMapper(eventPayload); + const update = provider.deltaMapper(payload); if (update.delta) { if (update.delta.added.length > 0) { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts index 500d30d992..0d910f664d 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts @@ -41,7 +41,7 @@ describe('WrapperProviders', () => { async databaseId => { const client = await databases.init(databaseId); - const provider1: IncrementalEntityProvider = { + const provider1: IncrementalEntityProvider = { getProviderName: () => 'provider1', around: burst => burst(0), next: async (_context, cursor) => { @@ -51,7 +51,7 @@ describe('WrapperProviders', () => { }, }; - const provider2: IncrementalEntityProvider = { + const provider2: IncrementalEntityProvider = { getProviderName: () => 'provider2', around: burst => burst(0), next: async (_context, cursor) => { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts index 6c82928bbf..2308db681b 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts @@ -58,7 +58,7 @@ export class WrapperProviders { ) {} wrap( - provider: IncrementalEntityProvider, + provider: IncrementalEntityProvider, options: IncrementalEntityProviderOptions, ): EntityProvider { this.numberOfProvidersToConnect += 1; @@ -81,8 +81,8 @@ export class WrapperProviders { ).createRouter(); } - private async startProvider( - provider: IncrementalEntityProvider, + private async startProvider( + provider: IncrementalEntityProvider, providerOptions: IncrementalEntityProviderOptions, connection: EntityProviderConnection, ) { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts index 06dfd1467c..d196693db8 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts @@ -24,7 +24,7 @@ import { incrementalIngestionEntityProviderCatalogModule } from './incrementalIn describe('bitbucketServerEntityProviderCatalogModule', () => { it('should register provider at the catalog extension point', async () => { - const provider1: IncrementalEntityProvider = { + const provider1: IncrementalEntityProvider = { getProviderName: () => 'provider1', around: burst => burst(0), next: async (cursor, _context) => { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts index 68f8bf1c90..2ed7dda728 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts @@ -38,7 +38,7 @@ export const incrementalIngestionEntityProviderCatalogModule = env, options: { providers: Array<{ - provider: IncrementalEntityProvider; + provider: IncrementalEntityProvider; options: IncrementalEntityProviderOptions; }>; }, diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts index 0f28dd7004..2dee5716c0 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts @@ -36,10 +36,10 @@ import { incrementalIngestionEntityProviderCatalogModule, } from '.'; -const provider: IncrementalEntityProvider = { +const provider: IncrementalEntityProvider = { getProviderName: () => 'test-provider', around: burst => burst(0), - next: async (_context, cursor: number | undefined) => { + next: async (_context, cursor) => { await new Promise(resolve => setTimeout(resolve, 500)); if (cursor === undefined || cursor < 3) { console.log(`### Returning batch #${cursor}`); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts index 875e77a7e9..e65698d4a4 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts @@ -68,8 +68,8 @@ export class IncrementalCatalogBuilder { return { incrementalAdminRouter }; } - addIncrementalEntityProvider( - provider: IncrementalEntityProvider, + addIncrementalEntityProvider( + provider: IncrementalEntityProvider, options: IncrementalEntityProviderOptions, ) { const { burstInterval, burstLength, restLength } = options; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts index acd5371573..f1325f5cc9 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts @@ -46,7 +46,7 @@ import { IncrementalIngestionDatabaseManager } from './database/IncrementalInges * * @public */ -export interface IncrementalEntityProvider { +export interface IncrementalEntityProvider { /** * This name must be unique between all of the entity providers * operating in the catalog. @@ -62,7 +62,10 @@ export interface IncrementalEntityProvider { * @returns The entities to be ingested, as well as the cursor of * the next page after this one. */ - next(context: unknown, cursor?: unknown): Promise; + next( + context: TContext, + cursor?: TCursor, + ): Promise>; /** * Do any setup and teardown necessary in order to provide the @@ -71,13 +74,13 @@ export interface IncrementalEntityProvider { * * @param burst - a function which performs a series of iterations */ - around(burst: (context: unknown) => Promise): Promise; + around(burst: (context: TContext) => Promise): Promise; /** * If present, this method maps incoming payloads to apply updates * outside of the incremental ingestion schedule. */ - deltaMapper?: (payload: unknown) => { + deltaMapper?: (payload: TInput) => { delta: | { added: DeferredEntity[]; @@ -93,16 +96,16 @@ export interface IncrementalEntityProvider { * * @public */ -export type EntityIteratorResult = +export type EntityIteratorResult = | { done: false; entities: DeferredEntity[]; - cursor: unknown; + cursor: T; } | { done: true; entities?: DeferredEntity[]; - cursor?: unknown; + cursor?: T; }; /** @public */ @@ -164,11 +167,11 @@ export interface IterationEngine { taskFn: TaskFunction; } -export interface IterationEngineOptions { +export interface IterationEngineOptions { logger: Logger; connection: EntityProviderConnection; manager: IncrementalIngestionDatabaseManager; - provider: IncrementalEntityProvider; + provider: IncrementalEntityProvider; restLength: DurationObjectUnits; ready: Promise; backoff?: IncrementalEntityProviderOptions['backoff'];