Merge pull request #19632 from backstage/freben/catalog-proc-int

allow app-config of the catalog processingInterval

plugins/catalog-backend/config.d.ts

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+import { HumanDuration } from '@backstage/types';
+
 export interface Config {
   /**
    * Configuration options for the catalog plugin.
@@ -142,5 +144,30 @@ export interface Config {
      * "keep".
      */
     orphanStrategy?: 'keep' | 'delete';
+
+    /**
+     * The interval at which the catalog should process its entities.
+     *
+     * @remarks
+     *
+     * Example:
+     *
+     * ```yaml
+     * catalog:
+     *   processingInterval: { minutes: 30 }
+     * ```
+     *
+     * Note that this is only a suggested minimum, and the actual interval may
+     * be longer. Internally, the catalog will scale up this number by a small
+     * factor and choose random numbers in that range to spread out the load. If
+     * the catalog is overloaded and cannot process all entities during the
+     * interval, the time taken between processing runs of any given entity may
+     * also be longer than specified here.
+     *
+     * Setting this value too low risks exhausting rate limits on external
+     * systems that are queried by processors, such as version control systems
+     * housing catalog-info files.
+     */
+    processingInterval?: HumanDuration;
   };
 }

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -74,7 +74,7 @@ import { createRouter } from './createRouter';
 import { DefaultRefreshService } from './DefaultRefreshService';
 import { AuthorizedRefreshService } from './AuthorizedRefreshService';
 import { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules';
-import { Config } from '@backstage/config';
+import { Config, readDurationFromConfig } from '@backstage/config';
 import { Logger } from 'winston';
 import { connectEntityProviders } from '../processing/connectEntityProviders';
 import { PermissionRuleParams } from '@backstage/plugin-permission-common';
@@ -100,6 +100,7 @@ import { AuthorizedLocationService } from './AuthorizedLocationService';
 import { DefaultProviderDatabase } from '../database/DefaultProviderDatabase';
 import { DefaultCatalogDatabase } from '../database/DefaultCatalogDatabase';
 import { EventBroker } from '@backstage/plugin-events-node';
+import { durationToMilliseconds } from '@backstage/types';
 
 /**
  * This is a duplicate of the alpha `CatalogPermissionRule` type, for use in the stable API.
@@ -160,11 +161,7 @@ export class CatalogBuilder {
     unprocessedEntity: Entity;
     errors: Error[];
   }) => Promise<void> | void;
-  private processingInterval: ProcessingIntervalFunction =
-    createRandomProcessingInterval({
-      minSeconds: 100,
-      maxSeconds: 150,
-    });
+  private processingInterval: ProcessingIntervalFunction;
   private locationAnalyzer: LocationAnalyzer | undefined = undefined;
   private readonly permissionRules: CatalogPermissionRuleInput[];
   private allowedLocationType: string[];
@@ -191,6 +188,10 @@ export class CatalogBuilder {
     this.parser = undefined;
     this.permissionRules = Object.values(catalogPermissionRules);
     this.allowedLocationType = ['url'];
+
+    this.processingInterval = CatalogBuilder.getDefaultProcessingInterval(
+      env.config,
+    );
   }
 
   /**
@@ -422,7 +423,7 @@ export class CatalogBuilder {
   }
 
   /**
-   * Enables the publishing of events for cloflicts in the DefaultProcessingDatabase
+   * Enables the publishing of events for conflicts in the DefaultProcessingDatabase
    */
   setEventBroker(broker: EventBroker): CatalogBuilder {
     this.eventBroker = broker;
@@ -758,4 +759,30 @@ export class CatalogBuilder {
       'https://backstage.io/docs/integrations/azure/org',
     );
   }
+
+  private static getDefaultProcessingInterval(
+    config: Config,
+  ): ProcessingIntervalFunction {
+    const processingIntervalKey = 'catalog.processingInterval';
+
+    if (!config.has(processingIntervalKey)) {
+      return createRandomProcessingInterval({
+        minSeconds: 100,
+        maxSeconds: 150,
+      });
+    }
+
+    const duration = readDurationFromConfig(config, {
+      key: processingIntervalKey,
+    });
+    const seconds = Math.max(
+      1,
+      Math.round(durationToMilliseconds(duration) / 1000),
+    );
+
+    return createRandomProcessingInterval({
+      minSeconds: seconds,
+      maxSeconds: seconds * 1.5,
+    });
+  }
 }