From 5f6c806378ba9899ab218a0dd6e9e253043e26ff Mon Sep 17 00:00:00 2001
From: Johan Haals <johan.haals@gmail.com>
Date: Fri, 11 Jun 2021 11:01:51 +0200
Subject: [PATCH] catalog/next: Make refresh interval configurable

Signed-off-by: Johan Haals <johan.haals@gmail.com>
---
 .../src/next/NextCatalogBuilder.ts              | 17 ++++++++++++++++-
 .../next/database/DefaultProcessingDatabase.ts  | 11 +++++++++--
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts
index 7a74576ea7..6685296312 100644
--- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts
+++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts
@@ -108,6 +108,7 @@ export class NextCatalogBuilder {
   private processors: CatalogProcessor[];
   private processorsReplace: boolean;
   private parser: CatalogProcessorParser | undefined;
+  private refreshIntervalSeconds = 100;
 
   constructor(env: CatalogEnvironment) {
     this.env = env;
@@ -136,6 +137,16 @@ export class NextCatalogBuilder {
     return this;
   }
 
+  /**
+   * Refresh interval determines how often entities should be refreshed.
+   * The default refresh duration is 100, setting this too low will potentially
+   * deplete request quotas to upstream services.
+   */
+  setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder {
+    this.refreshIntervalSeconds = seconds;
+    return this;
+  }
+
   /**
    * Sets what policies to use for validation of entities between the pre-
    * processing and post-processing stages. All such policies must pass for the
@@ -252,7 +263,11 @@ export class NextCatalogBuilder {
 
     const db = new CommonDatabase(dbClient, logger);
 
-    const processingDatabase = new DefaultProcessingDatabase(dbClient, logger);
+    const processingDatabase = new DefaultProcessingDatabase(
+      dbClient,
+      logger,
+      this.refreshIntervalSeconds,
+    );
     const integrations = ScmIntegrations.fromConfig(config);
     const orchestrator = new DefaultCatalogProcessingOrchestrator({
       processors,
diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts
index 3569d4dcc6..10c213d218 100644
--- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts
+++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts
@@ -46,6 +46,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
   constructor(
     private readonly database: Knex,
     private readonly logger: Logger,
+    private readonly refreshIntervalSeconds: number,
   ) {}
 
   async updateProcessedEntity(
@@ -377,8 +378,14 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
       .update({
         next_update_at:
           tx.client.config.client === 'sqlite3'
-            ? tx.raw(`datetime('now', ?)`, [`100 seconds`])
-            : tx.raw(`now() + interval '100 seconds'`),
+            ? tx.raw(`datetime('now', ?)`, [
+                `${this.refreshIntervalSeconds} seconds`,
+              ])
+            : tx.raw(
+                `now() + interval '${Number(
+                  this.refreshIntervalSeconds,
+                )} seconds'`,
+              ),
       });
 
     return {