catalog-backend: Add refresh interval function

Co-authored-by: Fredrik Adelöw <freben@users.noreply.github.com>
Signed-off-by: Johan Haals <johan.haals@gmail.com>

.changeset/early-ducks-stare.md

@@ -0,0 +1,8 @@
+---
+'@backstage/plugin-catalog-backend': minor
+---
+
+Updates the `DefaultProcessingDatabase` to accept a refresh interval function instead of a fixed refresh interval in seconds which used to default to 100s. The catalog now ships with a default refresh interval function that schedules entities for refresh every 100-150 seconds, this should
+help to smooth out bursts that occur when a lot of entities are scheduled for refresh at the same second.
+
+Custom `RefreshIntervalFunction` can be implemented and passed to the CatalogBuilder using `.setInterval(fn)`

plugins/catalog-backend/src/next/NextCatalogBuilder.ts

@@ -75,6 +75,10 @@ import { DefaultLocationStore } from './DefaultLocationStore';
 import { NextEntitiesCatalog } from './NextEntitiesCatalog';
 import { DefaultCatalogProcessingOrchestrator } from './processing/DefaultCatalogProcessingOrchestrator';
 import { Stitcher } from './stitching/Stitcher';
+import {
+  createRandomRefreshInterval,
+  RefreshIntervalFunction,
+} from './refresh';
 
 export type CatalogEnvironment = {
   logger: Logger;
@@ -112,8 +116,11 @@ export class NextCatalogBuilder {
   private processors: CatalogProcessor[];
   private processorsReplace: boolean;
   private parser: CatalogProcessorParser | undefined;
-  private refreshIntervalSeconds = 100;
-  private refreshSpreadSeconds = { min: 10, max: 60 };
+  private refreshInterval: RefreshIntervalFunction =
+    createRandomRefreshInterval({
+      minSeconds: 100,
+      maxSeconds: 150,
+    });
 
   constructor(env: CatalogEnvironment) {
     this.env = env;
@@ -145,11 +152,15 @@ export class NextCatalogBuilder {
 
   /**
    * Refresh interval determines how often entities should be refreshed.
-   * The default refresh duration is 100, setting this too low will potentially
-   * deplete request quotas to upstream services.
+   * Seconds provided will be multiplied by 1.5
+   * The default refresh duration is 100-150 seconds.
+   * setting this too low will potentially deplete request quotas to upstream services.
    */
   setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder {
-    this.refreshIntervalSeconds = seconds;
+    this.refreshInterval = createRandomRefreshInterval({
+      minSeconds: seconds,
+      maxSeconds: seconds * 1.5,
+    });
     return this;
   }
 
@@ -157,8 +168,10 @@ export class NextCatalogBuilder {
    * Refresh spread configures configures the minimum and maximum number of seconds
    * to wait between refreshes in addition to the configured refresh interval.
    */
-  setRefreshSpreadSeconds(min: number, max: number): NextCatalogBuilder {
-    this.refreshSpreadSeconds = { min, max };
+  setRefreshInterval(
+    refreshInterval: RefreshIntervalFunction,
+  ): NextCatalogBuilder {
+    this.refreshInterval = refreshInterval;
     return this;
   }
 
@@ -295,8 +308,7 @@ export class NextCatalogBuilder {
     const processingDatabase = new DefaultProcessingDatabase({
       database: dbClient,
       logger,
-      refreshIntervalSeconds: this.refreshIntervalSeconds,
-      refreshSpreadSeconds: this.refreshSpreadSeconds,
+      refreshInterval: this.refreshInterval,
     });
     const integrations = ScmIntegrations.fromConfig(config);
     const orchestrator = new DefaultCatalogProcessingOrchestrator({

plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts

@@ -32,6 +32,7 @@ import {
   DbRefreshStateRow,
   DbRelationsRow,
 } from './tables';
+import { createRandomRefreshInterval } from '../refresh';
 
 describe('Default Processing Database', () => {
   const defaultLogger = getVoidLogger();
@@ -50,8 +51,10 @@ describe('Default Processing Database', () => {
       db: new DefaultProcessingDatabase({
         database: knex,
         logger,
-        refreshIntervalSeconds: 100,
-        refreshSpreadSeconds: { min: 10, max: 60 },
+        refreshInterval: createRandomRefreshInterval({
+          minSeconds: 100,
+          maxSeconds: 150,
+        }),
       }),
     };
   }
@@ -999,7 +1002,7 @@ describe('Default Processing Database', () => {
           zone: 'utc',
         });
         expect(nextUpdate.diff(now, 'seconds').seconds).toBeGreaterThanOrEqual(
-          110,
+          100,
         );
       },
       60_000,

plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts

@@ -23,6 +23,7 @@ import { v4 as uuid } from 'uuid';
 import type { Logger } from 'winston';
 import { Transaction } from '../../database';
 import { DeferredEntity } from '../processing/types';
+import { RefreshIntervalFunction } from '../refresh';
 import {
   DbRefreshStateReferencesRow,
   DbRefreshStateRow,
@@ -48,11 +49,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
     private readonly options: {
       database: Knex;
       logger: Logger;
-      refreshIntervalSeconds: number;
-      refreshSpreadSeconds: {
-        min: number;
-        max: number;
-      };
+      refreshInterval: RefreshIntervalFunction;
     },
   ) {}
 
@@ -451,17 +448,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
     }
   }
 
-  // Returns the next update timestamp by combining refresh interval and refresh spread
-  private getNextUpdateAt(tx: Knex.Transaction): Knex.MaybeRawColumn<string> {
-    const { min, max } = this.options.refreshSpreadSeconds;
-    const refreshSpread = Math.floor(Math.random() * (max - min + 1)) + min;
-    const nextUpdate = this.options.refreshIntervalSeconds + refreshSpread;
-
-    return tx.client.config.client === 'sqlite3'
-      ? tx.raw(`datetime('now', ?)`, [`${nextUpdate} seconds`])
-      : tx.raw(`now() + interval '${Number(nextUpdate)} seconds'`);
-  }
-
   async getProcessableEntities(
     txOpaque: Transaction,
     request: { processBatchSize: number },
@@ -488,7 +474,14 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
         items.map(i => i.entity_ref),
       )
       .update({
-        next_update_at: this.getNextUpdateAt(tx),
+        next_update_at:
+          tx.client.config.client === 'sqlite3'
+            ? tx.raw(`datetime('now', ?)`, [
+                `${this.options.refreshInterval()} seconds`,
+              ])
+            : tx.raw(
+                `now() + interval '${this.options.refreshInterval()} seconds'`,
+              ),
       });
 
     return {

plugins/catalog-backend/src/next/index.ts

@@ -18,3 +18,5 @@ export { NextCatalogBuilder } from './NextCatalogBuilder';
 export { createNextRouter } from './NextRouter';
 export * from './processing';
 export * from './stitching';
+export type { RefreshIntervalFunction } from './refresh';
+export { createRandomRefreshInterval } from './refresh';

plugins/catalog-backend/src/next/refresh.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Function that returns the catalog refresh interval in seconds.
+ */
+export type RefreshIntervalFunction = () => number;
+
+/**
+ * @param {number} options.minSeconds The minimum number of seconds between refreshes
+ * @param {number} options.maxSeconds The maximum number of seconds between refreshes
+ * @returns {RefreshIntervalFunction} that provides the next refresh interval
+ *
+ */
+export function createRandomRefreshInterval(options: {
+  minSeconds: number;
+  maxSeconds: number;
+}): RefreshIntervalFunction {
+  const { minSeconds, maxSeconds } = options;
+  return () => {
+    return Math.random() * (maxSeconds - minSeconds) + minSeconds;
+  };
+}