diff --git a/.changeset/early-ducks-stare.md b/.changeset/early-ducks-stare.md new file mode 100644 index 0000000000..1fd44f2755 --- /dev/null +++ b/.changeset/early-ducks-stare.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Updates the `DefaultProcessingDatabase` to accept a refresh interval function instead of a fixed refresh interval in seconds which used to default to 100s. The catalog now ships with a default refresh interval function that schedules entities for refresh every 100-150 seconds, this should +help to smooth out bursts that occur when a lot of entities are scheduled for refresh at the same second. + +Custom `RefreshIntervalFunction` can be implemented and passed to the CatalogBuilder using `.setInterval(fn)` diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 7286855d4e..a6e67215c9 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -419,6 +419,16 @@ export function createNextRouter( options: RouterOptions_2, ): Promise; +// Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag +// Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" +// Warning: (ae-missing-release-tag) "createRandomRefreshInterval" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export function createRandomRefreshInterval(options: { + minSeconds: number; + maxSeconds: number; +}): RefreshIntervalFunction; + // Warning: (ae-missing-release-tag) "createRouter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -914,6 +924,9 @@ export class NextCatalogBuilder { key: string, resolver: PlaceholderResolver, ): NextCatalogBuilder; + setRefreshInterval( + refreshInterval: RefreshIntervalFunction, + ): NextCatalogBuilder; setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder; } @@ -987,6 +1000,11 @@ export type RecursivePartial = { : T[P]; }; +// Warning: (ae-missing-release-tag) "RefreshIntervalFunction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type RefreshIntervalFunction = () => number; + // Warning: (ae-missing-release-tag) "relation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -1107,8 +1125,8 @@ export class UrlReaderProcessor implements CatalogProcessor { // src/ingestion/types.d.ts:41:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/ingestion/types.d.ts:49:5 - (ae-forgotten-export) The symbol "AnalyzeLocationExistingEntity" needs to be exported by the entry point index.d.ts // src/ingestion/types.d.ts:50:5 - (ae-forgotten-export) The symbol "AnalyzeLocationGenerateEntity" needs to be exported by the entry point index.d.ts -// src/next/NextCatalogBuilder.d.ts:140:9 - (ae-forgotten-export) The symbol "CatalogProcessingEngine" needs to be exported by the entry point index.d.ts -// src/next/NextCatalogBuilder.d.ts:141:9 - (ae-forgotten-export) The symbol "LocationService" needs to be exported by the entry point index.d.ts +// src/next/NextCatalogBuilder.d.ts:147:9 - (ae-forgotten-export) The symbol "CatalogProcessingEngine" needs to be exported by the entry point index.d.ts +// src/next/NextCatalogBuilder.d.ts:148:9 - (ae-forgotten-export) The symbol "LocationService" needs to be exported by the entry point index.d.ts // src/next/processing/types.d.ts:11:5 - (ae-forgotten-export) The symbol "DeferredEntity" needs to be exported by the entry point index.d.ts // (No @packageDocumentation comment for this package) diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 0bf5a08dfd..f095f39785 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -73,7 +73,8 @@ "msw": "^0.29.0", "sqlite3": "^5.0.1", "supertest": "^6.1.3", - "wait-for-expect": "^3.0.2" + "wait-for-expect": "^3.0.2", + "luxon": "^2.0.2" }, "files": [ "dist", diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index fb12077508..675e170727 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -75,6 +75,10 @@ import { DefaultLocationStore } from './DefaultLocationStore'; import { NextEntitiesCatalog } from './NextEntitiesCatalog'; import { DefaultCatalogProcessingOrchestrator } from './processing/DefaultCatalogProcessingOrchestrator'; import { Stitcher } from './stitching/Stitcher'; +import { + createRandomRefreshInterval, + RefreshIntervalFunction, +} from './refresh'; export type CatalogEnvironment = { logger: Logger; @@ -112,7 +116,11 @@ export class NextCatalogBuilder { private processors: CatalogProcessor[]; private processorsReplace: boolean; private parser: CatalogProcessorParser | undefined; - private refreshIntervalSeconds = 100; + private refreshInterval: RefreshIntervalFunction = + createRandomRefreshInterval({ + minSeconds: 100, + maxSeconds: 150, + }); constructor(env: CatalogEnvironment) { this.env = env; @@ -144,11 +152,26 @@ export class NextCatalogBuilder { /** * Refresh interval determines how often entities should be refreshed. - * The default refresh duration is 100, setting this too low will potentially - * deplete request quotas to upstream services. + * Seconds provided will be multiplied by 1.5 + * The default refresh duration is 100-150 seconds. + * setting this too low will potentially deplete request quotas to upstream services. */ setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder { - this.refreshIntervalSeconds = seconds; + this.refreshInterval = createRandomRefreshInterval({ + minSeconds: seconds, + maxSeconds: seconds * 1.5, + }); + return this; + } + + /** + * Overwrites the default refresh interval function used to spread + * entity updates in the catalog. + */ + setRefreshInterval( + refreshInterval: RefreshIntervalFunction, + ): NextCatalogBuilder { + this.refreshInterval = refreshInterval; return this; } @@ -285,7 +308,7 @@ export class NextCatalogBuilder { const processingDatabase = new DefaultProcessingDatabase({ database: dbClient, logger, - refreshIntervalSeconds: this.refreshIntervalSeconds, + refreshInterval: this.refreshInterval, }); const integrations = ScmIntegrations.fromConfig(config); const orchestrator = new DefaultCatalogProcessingOrchestrator({ diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 5103c25dbc..df9c08ad6c 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -21,6 +21,7 @@ import { JsonObject } from '@backstage/config'; import { Knex } from 'knex'; import * as uuid from 'uuid'; import { Logger } from 'winston'; +import { DateTime } from 'luxon'; import { DatabaseManager } from './DatabaseManager'; import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; import { @@ -28,6 +29,7 @@ import { DbRefreshStateRow, DbRelationsRow, } from './tables'; +import { createRandomRefreshInterval } from '../refresh'; describe('Default Processing Database', () => { const defaultLogger = getVoidLogger(); @@ -46,7 +48,10 @@ describe('Default Processing Database', () => { db: new DefaultProcessingDatabase({ database: knex, logger, - refreshIntervalSeconds: 100, + refreshInterval: createRandomRefreshInterval({ + minSeconds: 100, + maxSeconds: 150, + }), }), }; } @@ -61,6 +66,21 @@ describe('Default Processing Database', () => { await db('refresh_state').insert(ref); }; + const parseDate = (date: string | Date): DateTime => { + const parsedDate = + typeof date === 'string' + ? DateTime.fromSQL(date, { zone: 'UTC' }) + : DateTime.fromJSDate(date); + + if (!parsedDate.isValid) { + throw new Error( + `Failed to parse date, reason: ${parsedDate.invalidReason}, explanation: ${parsedDate.invalidExplanation}`, + ); + } + + return parsedDate; + }; + describe('addUprocessedEntities', () => { function mockEntity(name: string, type: string): Entity { return { @@ -960,5 +980,42 @@ describe('Default Processing Database', () => { }, 60_000, ); + + it.each(databases.eachSupportedId())( + 'should update the next_refresh interval with a timestamp that includes refresh spread, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + const entity = JSON.stringify({ + kind: 'Location', + apiVersion: '1.0.0', + metadata: { + name: 'xyz', + }, + } as Entity); + await knex('refresh_state').insert({ + entity_id: '2', + entity_ref: 'location:default/new-root', + unprocessed_entity: entity, + errors: '[]', + next_update_at: '2019-01-01 23:00:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + await db.transaction(async tx => { + // Result does not include the updated timestamp + await db.getProcessableEntities(tx, { + processBatchSize: 1, + }); + }); + const now = DateTime.local(); + const result = await knex('refresh_state') + .where('entity_ref', 'location:default/new-root') + .select(); + const nextUpdate = parseDate(result[0].next_update_at); + expect(nextUpdate.diff(now, 'seconds').seconds).toBeGreaterThanOrEqual( + 100, + ); + }, + 60_000, + ); }); }); diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 177bf979ab..203ca3efbf 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -23,6 +23,7 @@ import { v4 as uuid } from 'uuid'; import type { Logger } from 'winston'; import { Transaction } from '../../database'; import { DeferredEntity } from '../processing/types'; +import { RefreshIntervalFunction } from '../refresh'; import { DbRefreshStateReferencesRow, DbRefreshStateRow, @@ -48,7 +49,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { private readonly options: { database: Knex; logger: Logger; - refreshIntervalSeconds: number; + refreshInterval: RefreshIntervalFunction; }, ) {} @@ -476,12 +477,10 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { next_update_at: tx.client.config.client === 'sqlite3' ? tx.raw(`datetime('now', ?)`, [ - `${this.options.refreshIntervalSeconds} seconds`, + `${this.options.refreshInterval()} seconds`, ]) : tx.raw( - `now() + interval '${Number( - this.options.refreshIntervalSeconds, - )} seconds'`, + `now() + interval '${this.options.refreshInterval()} seconds'`, ), }); diff --git a/plugins/catalog-backend/src/next/index.ts b/plugins/catalog-backend/src/next/index.ts index 8c1921ae8b..a27174c415 100644 --- a/plugins/catalog-backend/src/next/index.ts +++ b/plugins/catalog-backend/src/next/index.ts @@ -18,3 +18,5 @@ export { NextCatalogBuilder } from './NextCatalogBuilder'; export { createNextRouter } from './NextRouter'; export * from './processing'; export * from './stitching'; +export type { RefreshIntervalFunction } from './refresh'; +export { createRandomRefreshInterval } from './refresh'; diff --git a/plugins/catalog-backend/src/next/refresh.ts b/plugins/catalog-backend/src/next/refresh.ts new file mode 100644 index 0000000000..aae992f391 --- /dev/null +++ b/plugins/catalog-backend/src/next/refresh.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Function that returns the catalog refresh interval in seconds. + */ +export type RefreshIntervalFunction = () => number; + +/** + * Creates a function that returns a random refresh interval between minSeconds and maxSeconds. + * @returns {RefreshIntervalFunction} that provides the next refresh interval + */ +export function createRandomRefreshInterval(options: { + minSeconds: number; + maxSeconds: number; +}): RefreshIntervalFunction { + const { minSeconds, maxSeconds } = options; + return () => { + return Math.random() * (maxSeconds - minSeconds) + minSeconds; + }; +}