catalog/next: Make refresh interval configurable

Signed-off-by: Johan Haals <johan.haals@gmail.com>

plugins/catalog-backend/src/next/NextCatalogBuilder.ts

@@ -108,6 +108,7 @@ export class NextCatalogBuilder {
   private processors: CatalogProcessor[];
   private processorsReplace: boolean;
   private parser: CatalogProcessorParser | undefined;
+  private refreshIntervalSeconds = 100;
 
   constructor(env: CatalogEnvironment) {
     this.env = env;
@@ -136,6 +137,16 @@ export class NextCatalogBuilder {
     return this;
   }
 
+  /**
+   * Refresh interval determines how often entities should be refreshed.
+   * The default refresh duration is 100, setting this too low will potentially
+   * deplete request quotas to upstream services.
+   */
+  setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder {
+    this.refreshIntervalSeconds = seconds;
+    return this;
+  }
+
   /**
    * Sets what policies to use for validation of entities between the pre-
    * processing and post-processing stages. All such policies must pass for the
@@ -252,7 +263,11 @@ export class NextCatalogBuilder {
 
     const db = new CommonDatabase(dbClient, logger);
 
-    const processingDatabase = new DefaultProcessingDatabase(dbClient, logger);
+    const processingDatabase = new DefaultProcessingDatabase(
+      dbClient,
+      logger,
+      this.refreshIntervalSeconds,
+    );
     const integrations = ScmIntegrations.fromConfig(config);
     const orchestrator = new DefaultCatalogProcessingOrchestrator({
       processors,

plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts

@@ -46,6 +46,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
   constructor(
     private readonly database: Knex,
     private readonly logger: Logger,
+    private readonly refreshIntervalSeconds: number,
   ) {}
 
   async updateProcessedEntity(
@@ -377,8 +378,14 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
       .update({
         next_update_at:
           tx.client.config.client === 'sqlite3'
-            ? tx.raw(`datetime('now', ?)`, [`100 seconds`])
-            : tx.raw(`now() + interval '100 seconds'`),
+            ? tx.raw(`datetime('now', ?)`, [
+                `${this.refreshIntervalSeconds} seconds`,
+              ])
+            : tx.raw(
+                `now() + interval '${Number(
+                  this.refreshIntervalSeconds,
+                )} seconds'`,
+              ),
       });
 
     return {