Merge pull request #6007 from backstage/jhaals/configure-refresh

catalog/next: Make refresh interval configurable

plugins/catalog-backend/src/next/NextCatalogBuilder.ts

@@ -108,6 +108,7 @@ export class NextCatalogBuilder {
   private processors: CatalogProcessor[];
   private processorsReplace: boolean;
   private parser: CatalogProcessorParser | undefined;
+  private refreshIntervalSeconds = 100;
 
   constructor(env: CatalogEnvironment) {
     this.env = env;
@@ -136,6 +137,16 @@ export class NextCatalogBuilder {
     return this;
   }
 
+  /**
+   * Refresh interval determines how often entities should be refreshed.
+   * The default refresh duration is 100, setting this too low will potentially
+   * deplete request quotas to upstream services.
+   */
+  setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder {
+    this.refreshIntervalSeconds = seconds;
+    return this;
+  }
+
   /**
    * Sets what policies to use for validation of entities between the pre-
    * processing and post-processing stages. All such policies must pass for the
@@ -252,7 +263,11 @@ export class NextCatalogBuilder {
 
     const db = new CommonDatabase(dbClient, logger);
 
-    const processingDatabase = new DefaultProcessingDatabase(dbClient, logger);
+    const processingDatabase = new DefaultProcessingDatabase({
+      database: dbClient,
+      logger,
+      refreshIntervalSeconds: this.refreshIntervalSeconds,
+    });
     const integrations = ScmIntegrations.fromConfig(config);
     const orchestrator = new DefaultCatalogProcessingOrchestrator({
       processors,

plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts

@@ -39,7 +39,11 @@ describe('Default Processing Database', () => {
     await DatabaseManager.createDatabase(knex);
     return {
       knex,
-      db: new DefaultProcessingDatabase(knex, logger),
+      db: new DefaultProcessingDatabase({
+        database: knex,
+        logger,
+        refreshIntervalSeconds: 100,
+      }),
     };
   }
 

plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts

@@ -44,8 +44,11 @@ const BATCH_SIZE = 50;
 
 export class DefaultProcessingDatabase implements ProcessingDatabase {
   constructor(
-    private readonly database: Knex,
-    private readonly logger: Logger,
+    private readonly options: {
+      database: Knex;
+      logger: Logger;
+      refreshIntervalSeconds: number;
+    },
   ) {}
 
   async updateProcessedEntity(
@@ -272,7 +275,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
         .whereIn('target_entity_ref', toRemove)
         .delete();
 
-      this.logger.debug(
+      this.options.logger.debug(
         `removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`,
       );
     }
@@ -377,8 +380,14 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
       .update({
         next_update_at:
           tx.client.config.client === 'sqlite3'
-            ? tx.raw(`datetime('now', ?)`, [`100 seconds`])
-            : tx.raw(`now() + interval '100 seconds'`),
+            ? tx.raw(`datetime('now', ?)`, [
+                `${this.options.refreshIntervalSeconds} seconds`,
+              ])
+            : tx.raw(
+                `now() + interval '${Number(
+                  this.options.refreshIntervalSeconds,
+                )} seconds'`,
+              ),
       });
 
     return {
@@ -406,7 +415,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
     try {
       let result: T | undefined = undefined;
 
-      await this.database.transaction(
+      await this.options.database.transaction(
         async tx => {
           // We can't return here, as knex swallows the return type in case the transaction is rolled back:
           // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136
@@ -420,7 +429,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
 
       return result!;
     } catch (e) {
-      this.logger.debug(`Error during transaction, ${e}`);
+      this.options.logger.debug(`Error during transaction, ${e}`);
 
       if (
         /SQLITE_CONSTRAINT: UNIQUE/.test(e.message) ||