From fe960ad0fd60929a2997771d73c049ef404a0e0b Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 13 Aug 2021 15:31:07 +0200 Subject: [PATCH] catalog-backend: Add refresh interval function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: Johan Haals --- .changeset/early-ducks-stare.md | 8 +++++ .../src/next/NextCatalogBuilder.ts | 30 +++++++++++----- .../DefaultProcessingDatabase.test.ts | 9 +++-- .../database/DefaultProcessingDatabase.ts | 27 ++++++-------- plugins/catalog-backend/src/next/index.ts | 2 ++ plugins/catalog-backend/src/next/refresh.ts | 36 +++++++++++++++++++ 6 files changed, 83 insertions(+), 29 deletions(-) create mode 100644 .changeset/early-ducks-stare.md create mode 100644 plugins/catalog-backend/src/next/refresh.ts diff --git a/.changeset/early-ducks-stare.md b/.changeset/early-ducks-stare.md new file mode 100644 index 0000000000..58b1525e25 --- /dev/null +++ b/.changeset/early-ducks-stare.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Updates the `DefaultProcessingDatabase` to accept a refresh interval function instead of a fixed refresh interval in seconds which used to default to 100s. The catalog now ships with a default refresh interval function that schedules entities for refresh every 100-150 seconds, this should +help to smooth out bursts that occur when a lot of entities are scheduled for refresh at the same second. + +Custom `RefreshIntervalFunction` can be implemented and passed to the CatalogBuilder using `.setInterval(fn)` diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index fe7df180ee..b277f0b16a 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -75,6 +75,10 @@ import { DefaultLocationStore } from './DefaultLocationStore'; import { NextEntitiesCatalog } from './NextEntitiesCatalog'; import { DefaultCatalogProcessingOrchestrator } from './processing/DefaultCatalogProcessingOrchestrator'; import { Stitcher } from './stitching/Stitcher'; +import { + createRandomRefreshInterval, + RefreshIntervalFunction, +} from './refresh'; export type CatalogEnvironment = { logger: Logger; @@ -112,8 +116,11 @@ export class NextCatalogBuilder { private processors: CatalogProcessor[]; private processorsReplace: boolean; private parser: CatalogProcessorParser | undefined; - private refreshIntervalSeconds = 100; - private refreshSpreadSeconds = { min: 10, max: 60 }; + private refreshInterval: RefreshIntervalFunction = + createRandomRefreshInterval({ + minSeconds: 100, + maxSeconds: 150, + }); constructor(env: CatalogEnvironment) { this.env = env; @@ -145,11 +152,15 @@ export class NextCatalogBuilder { /** * Refresh interval determines how often entities should be refreshed. - * The default refresh duration is 100, setting this too low will potentially - * deplete request quotas to upstream services. + * Seconds provided will be multiplied by 1.5 + * The default refresh duration is 100-150 seconds. + * setting this too low will potentially deplete request quotas to upstream services. */ setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder { - this.refreshIntervalSeconds = seconds; + this.refreshInterval = createRandomRefreshInterval({ + minSeconds: seconds, + maxSeconds: seconds * 1.5, + }); return this; } @@ -157,8 +168,10 @@ export class NextCatalogBuilder { * Refresh spread configures configures the minimum and maximum number of seconds * to wait between refreshes in addition to the configured refresh interval. */ - setRefreshSpreadSeconds(min: number, max: number): NextCatalogBuilder { - this.refreshSpreadSeconds = { min, max }; + setRefreshInterval( + refreshInterval: RefreshIntervalFunction, + ): NextCatalogBuilder { + this.refreshInterval = refreshInterval; return this; } @@ -295,8 +308,7 @@ export class NextCatalogBuilder { const processingDatabase = new DefaultProcessingDatabase({ database: dbClient, logger, - refreshIntervalSeconds: this.refreshIntervalSeconds, - refreshSpreadSeconds: this.refreshSpreadSeconds, + refreshInterval: this.refreshInterval, }); const integrations = ScmIntegrations.fromConfig(config); const orchestrator = new DefaultCatalogProcessingOrchestrator({ diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 690dcc1b7b..5b717ee36e 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -32,6 +32,7 @@ import { DbRefreshStateRow, DbRelationsRow, } from './tables'; +import { createRandomRefreshInterval } from '../refresh'; describe('Default Processing Database', () => { const defaultLogger = getVoidLogger(); @@ -50,8 +51,10 @@ describe('Default Processing Database', () => { db: new DefaultProcessingDatabase({ database: knex, logger, - refreshIntervalSeconds: 100, - refreshSpreadSeconds: { min: 10, max: 60 }, + refreshInterval: createRandomRefreshInterval({ + minSeconds: 100, + maxSeconds: 150, + }), }), }; } @@ -999,7 +1002,7 @@ describe('Default Processing Database', () => { zone: 'utc', }); expect(nextUpdate.diff(now, 'seconds').seconds).toBeGreaterThanOrEqual( - 110, + 100, ); }, 60_000, diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 926423c848..203ca3efbf 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -23,6 +23,7 @@ import { v4 as uuid } from 'uuid'; import type { Logger } from 'winston'; import { Transaction } from '../../database'; import { DeferredEntity } from '../processing/types'; +import { RefreshIntervalFunction } from '../refresh'; import { DbRefreshStateReferencesRow, DbRefreshStateRow, @@ -48,11 +49,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { private readonly options: { database: Knex; logger: Logger; - refreshIntervalSeconds: number; - refreshSpreadSeconds: { - min: number; - max: number; - }; + refreshInterval: RefreshIntervalFunction; }, ) {} @@ -451,17 +448,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { } } - // Returns the next update timestamp by combining refresh interval and refresh spread - private getNextUpdateAt(tx: Knex.Transaction): Knex.MaybeRawColumn { - const { min, max } = this.options.refreshSpreadSeconds; - const refreshSpread = Math.floor(Math.random() * (max - min + 1)) + min; - const nextUpdate = this.options.refreshIntervalSeconds + refreshSpread; - - return tx.client.config.client === 'sqlite3' - ? tx.raw(`datetime('now', ?)`, [`${nextUpdate} seconds`]) - : tx.raw(`now() + interval '${Number(nextUpdate)} seconds'`); - } - async getProcessableEntities( txOpaque: Transaction, request: { processBatchSize: number }, @@ -488,7 +474,14 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { items.map(i => i.entity_ref), ) .update({ - next_update_at: this.getNextUpdateAt(tx), + next_update_at: + tx.client.config.client === 'sqlite3' + ? tx.raw(`datetime('now', ?)`, [ + `${this.options.refreshInterval()} seconds`, + ]) + : tx.raw( + `now() + interval '${this.options.refreshInterval()} seconds'`, + ), }); return { diff --git a/plugins/catalog-backend/src/next/index.ts b/plugins/catalog-backend/src/next/index.ts index 8c1921ae8b..a27174c415 100644 --- a/plugins/catalog-backend/src/next/index.ts +++ b/plugins/catalog-backend/src/next/index.ts @@ -18,3 +18,5 @@ export { NextCatalogBuilder } from './NextCatalogBuilder'; export { createNextRouter } from './NextRouter'; export * from './processing'; export * from './stitching'; +export type { RefreshIntervalFunction } from './refresh'; +export { createRandomRefreshInterval } from './refresh'; diff --git a/plugins/catalog-backend/src/next/refresh.ts b/plugins/catalog-backend/src/next/refresh.ts new file mode 100644 index 0000000000..d9f56c3254 --- /dev/null +++ b/plugins/catalog-backend/src/next/refresh.ts @@ -0,0 +1,36 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Function that returns the catalog refresh interval in seconds. + */ +export type RefreshIntervalFunction = () => number; + +/** + * @param {number} options.minSeconds The minimum number of seconds between refreshes + * @param {number} options.maxSeconds The maximum number of seconds between refreshes + * @returns {RefreshIntervalFunction} that provides the next refresh interval + * + */ +export function createRandomRefreshInterval(options: { + minSeconds: number; + maxSeconds: number; +}): RefreshIntervalFunction { + const { minSeconds, maxSeconds } = options; + return () => { + return Math.random() * (maxSeconds - minSeconds) + minSeconds; + }; +}