diff --git a/.changeset/kind-tools-kneel.md b/.changeset/kind-tools-kneel.md new file mode 100644 index 0000000000..6296ac7011 --- /dev/null +++ b/.changeset/kind-tools-kneel.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Make refresh interval configurable for the `NextCatalogBuilder` using `.setRefreshIntervalSeconds()`. + +Change `DefaultProcessingDatabase` constructor to accept an options object instead of individual arguments. diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 7a74576ea7..944036e48c 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -108,6 +108,7 @@ export class NextCatalogBuilder { private processors: CatalogProcessor[]; private processorsReplace: boolean; private parser: CatalogProcessorParser | undefined; + private refreshIntervalSeconds = 100; constructor(env: CatalogEnvironment) { this.env = env; @@ -136,6 +137,16 @@ export class NextCatalogBuilder { return this; } + /** + * Refresh interval determines how often entities should be refreshed. + * The default refresh duration is 100, setting this too low will potentially + * deplete request quotas to upstream services. + */ + setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder { + this.refreshIntervalSeconds = seconds; + return this; + } + /** * Sets what policies to use for validation of entities between the pre- * processing and post-processing stages. All such policies must pass for the @@ -252,7 +263,11 @@ export class NextCatalogBuilder { const db = new CommonDatabase(dbClient, logger); - const processingDatabase = new DefaultProcessingDatabase(dbClient, logger); + const processingDatabase = new DefaultProcessingDatabase({ + database: dbClient, + logger, + refreshIntervalSeconds: this.refreshIntervalSeconds, + }); const integrations = ScmIntegrations.fromConfig(config); const orchestrator = new DefaultCatalogProcessingOrchestrator({ processors, diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 7d98a32377..0f83311944 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -39,7 +39,11 @@ describe('Default Processing Database', () => { await DatabaseManager.createDatabase(knex); return { knex, - db: new DefaultProcessingDatabase(knex, logger), + db: new DefaultProcessingDatabase({ + database: knex, + logger, + refreshIntervalSeconds: 100, + }), }; } diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 3569d4dcc6..f4804f9118 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -44,8 +44,11 @@ const BATCH_SIZE = 50; export class DefaultProcessingDatabase implements ProcessingDatabase { constructor( - private readonly database: Knex, - private readonly logger: Logger, + private readonly options: { + database: Knex; + logger: Logger; + refreshIntervalSeconds: number; + }, ) {} async updateProcessedEntity( @@ -272,7 +275,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { .whereIn('target_entity_ref', toRemove) .delete(); - this.logger.debug( + this.options.logger.debug( `removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`, ); } @@ -377,8 +380,14 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { .update({ next_update_at: tx.client.config.client === 'sqlite3' - ? tx.raw(`datetime('now', ?)`, [`100 seconds`]) - : tx.raw(`now() + interval '100 seconds'`), + ? tx.raw(`datetime('now', ?)`, [ + `${this.options.refreshIntervalSeconds} seconds`, + ]) + : tx.raw( + `now() + interval '${Number( + this.options.refreshIntervalSeconds, + )} seconds'`, + ), }); return { @@ -406,7 +415,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { try { let result: T | undefined = undefined; - await this.database.transaction( + await this.options.database.transaction( async tx => { // We can't return here, as knex swallows the return type in case the transaction is rolled back: // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136 @@ -420,7 +429,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { return result!; } catch (e) { - this.logger.debug(`Error during transaction, ${e}`); + this.options.logger.debug(`Error during transaction, ${e}`); if ( /SQLITE_CONSTRAINT: UNIQUE/.test(e.message) ||