From 880967948080d2bb56376be17265cc6ae70e41f7 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Mar 2022 10:21:46 +0100 Subject: [PATCH] chore: renaming refreshInterval to processingInterval Signed-off-by: blam --- .../DefaultProcessingDatabase.test.ts | 4 +- .../catalog-backend/src/processing/index.ts | 10 ++++- .../catalog-backend/src/processing/refresh.ts | 23 ++++++++++ .../src/service/CatalogBuilder.ts | 42 ++++++++++++++++--- 4 files changed, 69 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 729be442aa..c7cd6b7026 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -28,7 +28,7 @@ import { DbRefreshStateRow, DbRelationsRow, } from './tables'; -import { createRandomRefreshInterval } from '../processing/refresh'; +import { createRandomProcessingInterval } from '../processing/refresh'; import { timestampToDateTime } from './conversion'; import { generateStableHash } from './util'; @@ -49,7 +49,7 @@ describe('Default Processing Database', () => { db: new DefaultProcessingDatabase({ database: knex, logger, - refreshInterval: createRandomRefreshInterval({ + refreshInterval: createRandomProcessingInterval({ minSeconds: 100, maxSeconds: 150, }), diff --git a/plugins/catalog-backend/src/processing/index.ts b/plugins/catalog-backend/src/processing/index.ts index dc8b07eceb..aec6a5d71c 100644 --- a/plugins/catalog-backend/src/processing/index.ts +++ b/plugins/catalog-backend/src/processing/index.ts @@ -23,5 +23,11 @@ export type { } from './types'; export { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator'; -export { createRandomRefreshInterval } from './refresh'; -export type { RefreshIntervalFunction } from './refresh'; +export { + createRandomRefreshInterval, + createRandomProcessingInterval, +} from './refresh'; +export type { + RefreshIntervalFunction, + ProcessingIntervalFunction, +} from './refresh'; diff --git a/plugins/catalog-backend/src/processing/refresh.ts b/plugins/catalog-backend/src/processing/refresh.ts index 3e03e3add8..941c338912 100644 --- a/plugins/catalog-backend/src/processing/refresh.ts +++ b/plugins/catalog-backend/src/processing/refresh.ts @@ -16,13 +16,21 @@ /** * Function that returns the catalog refresh interval in seconds. + * @deprecated use {@link ProcessingIntervalFunction} instead * @public */ export type RefreshIntervalFunction = () => number; +/** + * Function that returns the catalog processing interval in seconds. + * @public + */ +export type ProcessingIntervalFunction = () => number; + /** * Creates a function that returns a random refresh interval between minSeconds and maxSeconds. * @returns A {@link RefreshIntervalFunction} that provides the next refresh interval + * @deprecated use {@link createRandomProcessingInterval} instead * @public */ export function createRandomRefreshInterval(options: { @@ -34,3 +42,18 @@ export function createRandomRefreshInterval(options: { return Math.random() * (maxSeconds - minSeconds) + minSeconds; }; } + +/** + * Creates a function that returns a random processing interval between minSeconds and maxSeconds. + * @returns A {@link ProcessingIntervalFunction} that provides the next processing interval + * @public + */ +export function createRandomProcessingInterval(options: { + minSeconds: number; + maxSeconds: number; +}): ProcessingIntervalFunction { + const { minSeconds, maxSeconds } = options; + return () => { + return Math.random() * (maxSeconds - minSeconds) + minSeconds; + }; +} diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index f62377decb..ce55bc5937 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -76,8 +76,9 @@ import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog'; import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; import { Stitcher } from '../stitching/Stitcher'; import { - createRandomRefreshInterval, + createRandomProcessingInterval, RefreshIntervalFunction, + ProcessingIntervalFunction, } from '../processing/refresh'; import { createRouter } from './createRouter'; import { DefaultRefreshService } from './DefaultRefreshService'; @@ -139,8 +140,8 @@ export class CatalogBuilder { private processors: CatalogProcessor[]; private processorsReplace: boolean; private parser: CatalogProcessorParser | undefined; - private refreshInterval: RefreshIntervalFunction = - createRandomRefreshInterval({ + private processingInterval: ProcessingIntervalFunction = + createRandomProcessingInterval({ minSeconds: 100, maxSeconds: 150, }); @@ -192,9 +193,25 @@ export class CatalogBuilder { * Seconds provided will be multiplied by 1.5 * The default refresh duration is 100-150 seconds. * setting this too low will potentially deplete request quotas to upstream services. + * + * @deprecated use {@link CatalogBuilder#setProcessingIntervalSeconds} instead */ setRefreshIntervalSeconds(seconds: number): CatalogBuilder { - this.refreshInterval = createRandomRefreshInterval({ + this.processingInterval = createRandomProcessingInterval({ + minSeconds: seconds, + maxSeconds: seconds * 1.5, + }); + return this; + } + + /** + * Processing interval determines how often entities should be processed. + * Seconds provided will be multiplied by 1.5 + * The default processing duration is 100-150 seconds. + * setting this too low will potentially deplete request quotas to upstream services. + */ + setProcessingIntervalSeconds(seconds: number): CatalogBuilder { + this.processingInterval = createRandomProcessingInterval({ minSeconds: seconds, maxSeconds: seconds * 1.5, }); @@ -204,9 +221,22 @@ export class CatalogBuilder { /** * Overwrites the default refresh interval function used to spread * entity updates in the catalog. + * + * @deprecated use {@link CatalogBuilder#setProcessingInterval} instead */ setRefreshInterval(refreshInterval: RefreshIntervalFunction): CatalogBuilder { - this.refreshInterval = refreshInterval; + this.processingInterval = refreshInterval; + return this; + } + + /** + * Overwrites the default processing interval function used to spread + * entity updates in the catalog. + */ + setProcessingInterval( + processingInterval: ProcessingIntervalFunction, + ): CatalogBuilder { + this.processingInterval = processingInterval; return this; } @@ -396,7 +426,7 @@ export class CatalogBuilder { const processingDatabase = new DefaultProcessingDatabase({ database: dbClient, logger, - refreshInterval: this.refreshInterval, + refreshInterval: this.processingInterval, }); const integrations = ScmIntegrations.fromConfig(config); const rulesEnforcer = DefaultCatalogRulesEnforcer.fromConfig(config);