Merge pull request #6822 from backstage/jhaals/refresh-timer

catalog-backend: Add refresh interval function

.changeset/early-ducks-stare.md

@@ -0,0 +1,8 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Updates the `DefaultProcessingDatabase` to accept a refresh interval function instead of a fixed refresh interval in seconds which used to default to 100s. The catalog now ships with a default refresh interval function that schedules entities for refresh every 100-150 seconds, this should
+help to smooth out bursts that occur when a lot of entities are scheduled for refresh at the same second.
+
+Custom `RefreshIntervalFunction` can be implemented and passed to the CatalogBuilder using `.setInterval(fn)`

plugins/catalog-backend/api-report.md

@@ -419,6 +419,16 @@ export function createNextRouter(
   options: RouterOptions_2,
 ): Promise<express.Router>;
 
+// Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag
+// Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@"
+// Warning: (ae-missing-release-tag) "createRandomRefreshInterval" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public
+export function createRandomRefreshInterval(options: {
+  minSeconds: number;
+  maxSeconds: number;
+}): RefreshIntervalFunction;
+
 // Warning: (ae-missing-release-tag) "createRouter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -914,6 +924,9 @@ export class NextCatalogBuilder {
     key: string,
     resolver: PlaceholderResolver,
   ): NextCatalogBuilder;
+  setRefreshInterval(
+    refreshInterval: RefreshIntervalFunction,
+  ): NextCatalogBuilder;
   setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder;
 }
 
@@ -987,6 +1000,11 @@ export type RecursivePartial<T> = {
     : T[P];
 };
 
+// Warning: (ae-missing-release-tag) "RefreshIntervalFunction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public
+export type RefreshIntervalFunction = () => number;
+
 // Warning: (ae-missing-release-tag) "relation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -1107,8 +1125,8 @@ export class UrlReaderProcessor implements CatalogProcessor {
 // src/ingestion/types.d.ts:41:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
 // src/ingestion/types.d.ts:49:5 - (ae-forgotten-export) The symbol "AnalyzeLocationExistingEntity" needs to be exported by the entry point index.d.ts
 // src/ingestion/types.d.ts:50:5 - (ae-forgotten-export) The symbol "AnalyzeLocationGenerateEntity" needs to be exported by the entry point index.d.ts
-// src/next/NextCatalogBuilder.d.ts:140:9 - (ae-forgotten-export) The symbol "CatalogProcessingEngine" needs to be exported by the entry point index.d.ts
-// src/next/NextCatalogBuilder.d.ts:141:9 - (ae-forgotten-export) The symbol "LocationService" needs to be exported by the entry point index.d.ts
+// src/next/NextCatalogBuilder.d.ts:147:9 - (ae-forgotten-export) The symbol "CatalogProcessingEngine" needs to be exported by the entry point index.d.ts
+// src/next/NextCatalogBuilder.d.ts:148:9 - (ae-forgotten-export) The symbol "LocationService" needs to be exported by the entry point index.d.ts
 // src/next/processing/types.d.ts:11:5 - (ae-forgotten-export) The symbol "DeferredEntity" needs to be exported by the entry point index.d.ts
 
 // (No @packageDocumentation comment for this package)

plugins/catalog-backend/package.json

@@ -73,7 +73,8 @@
     "msw": "^0.29.0",
     "sqlite3": "^5.0.1",
     "supertest": "^6.1.3",
-    "wait-for-expect": "^3.0.2"
+    "wait-for-expect": "^3.0.2",
+    "luxon": "^2.0.2"
   },
   "files": [
     "dist",

plugins/catalog-backend/src/next/NextCatalogBuilder.ts

@@ -75,6 +75,10 @@ import { DefaultLocationStore } from './DefaultLocationStore';
 import { NextEntitiesCatalog } from './NextEntitiesCatalog';
 import { DefaultCatalogProcessingOrchestrator } from './processing/DefaultCatalogProcessingOrchestrator';
 import { Stitcher } from './stitching/Stitcher';
+import {
+  createRandomRefreshInterval,
+  RefreshIntervalFunction,
+} from './refresh';
 
 export type CatalogEnvironment = {
   logger: Logger;
@@ -112,7 +116,11 @@ export class NextCatalogBuilder {
   private processors: CatalogProcessor[];
   private processorsReplace: boolean;
   private parser: CatalogProcessorParser | undefined;
-  private refreshIntervalSeconds = 100;
+  private refreshInterval: RefreshIntervalFunction =
+    createRandomRefreshInterval({
+      minSeconds: 100,
+      maxSeconds: 150,
+    });
 
   constructor(env: CatalogEnvironment) {
     this.env = env;
@@ -144,11 +152,26 @@ export class NextCatalogBuilder {
 
   /**
    * Refresh interval determines how often entities should be refreshed.
-   * The default refresh duration is 100, setting this too low will potentially
-   * deplete request quotas to upstream services.
+   * Seconds provided will be multiplied by 1.5
+   * The default refresh duration is 100-150 seconds.
+   * setting this too low will potentially deplete request quotas to upstream services.
    */
   setRefreshIntervalSeconds(seconds: number): NextCatalogBuilder {
-    this.refreshIntervalSeconds = seconds;
+    this.refreshInterval = createRandomRefreshInterval({
+      minSeconds: seconds,
+      maxSeconds: seconds * 1.5,
+    });
+    return this;
+  }
+
+  /**
+   * Overwrites the default refresh interval function used to spread
+   * entity updates in the catalog.
+   */
+  setRefreshInterval(
+    refreshInterval: RefreshIntervalFunction,
+  ): NextCatalogBuilder {
+    this.refreshInterval = refreshInterval;
     return this;
   }
 
@@ -285,7 +308,7 @@ export class NextCatalogBuilder {
     const processingDatabase = new DefaultProcessingDatabase({
       database: dbClient,
       logger,
-      refreshIntervalSeconds: this.refreshIntervalSeconds,
+      refreshInterval: this.refreshInterval,
     });
     const integrations = ScmIntegrations.fromConfig(config);
     const orchestrator = new DefaultCatalogProcessingOrchestrator({

plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts

@@ -21,6 +21,7 @@ import { JsonObject } from '@backstage/config';
 import { Knex } from 'knex';
 import * as uuid from 'uuid';
 import { Logger } from 'winston';
+import { DateTime } from 'luxon';
 import { DatabaseManager } from './DatabaseManager';
 import { DefaultProcessingDatabase } from './DefaultProcessingDatabase';
 import {
@@ -28,6 +29,7 @@ import {
   DbRefreshStateRow,
   DbRelationsRow,
 } from './tables';
+import { createRandomRefreshInterval } from '../refresh';
 
 describe('Default Processing Database', () => {
   const defaultLogger = getVoidLogger();
@@ -46,7 +48,10 @@ describe('Default Processing Database', () => {
       db: new DefaultProcessingDatabase({
         database: knex,
         logger,
-        refreshIntervalSeconds: 100,
+        refreshInterval: createRandomRefreshInterval({
+          minSeconds: 100,
+          maxSeconds: 150,
+        }),
       }),
     };
   }
@@ -61,6 +66,21 @@ describe('Default Processing Database', () => {
     await db<DbRefreshStateRow>('refresh_state').insert(ref);
   };
 
+  const parseDate = (date: string | Date): DateTime => {
+    const parsedDate =
+      typeof date === 'string'
+        ? DateTime.fromSQL(date, { zone: 'UTC' })
+        : DateTime.fromJSDate(date);
+
+    if (!parsedDate.isValid) {
+      throw new Error(
+        `Failed to parse date, reason: ${parsedDate.invalidReason}, explanation: ${parsedDate.invalidExplanation}`,
+      );
+    }
+
+    return parsedDate;
+  };
+
   describe('addUprocessedEntities', () => {
     function mockEntity(name: string, type: string): Entity {
       return {
@@ -960,5 +980,42 @@ describe('Default Processing Database', () => {
       },
       60_000,
     );
+
+    it.each(databases.eachSupportedId())(
+      'should update the next_refresh interval with a timestamp that includes refresh spread, %p',
+      async databaseId => {
+        const { knex, db } = await createDatabase(databaseId);
+        const entity = JSON.stringify({
+          kind: 'Location',
+          apiVersion: '1.0.0',
+          metadata: {
+            name: 'xyz',
+          },
+        } as Entity);
+        await knex<DbRefreshStateRow>('refresh_state').insert({
+          entity_id: '2',
+          entity_ref: 'location:default/new-root',
+          unprocessed_entity: entity,
+          errors: '[]',
+          next_update_at: '2019-01-01 23:00:00',
+          last_discovery_at: '2021-04-01 13:37:00',
+        });
+        await db.transaction(async tx => {
+          // Result does not include the updated timestamp
+          await db.getProcessableEntities(tx, {
+            processBatchSize: 1,
+          });
+        });
+        const now = DateTime.local();
+        const result = await knex<DbRefreshStateRow>('refresh_state')
+          .where('entity_ref', 'location:default/new-root')
+          .select();
+        const nextUpdate = parseDate(result[0].next_update_at);
+        expect(nextUpdate.diff(now, 'seconds').seconds).toBeGreaterThanOrEqual(
+          100,
+        );
+      },
+      60_000,
+    );
   });
 });

plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts

@@ -23,6 +23,7 @@ import { v4 as uuid } from 'uuid';
 import type { Logger } from 'winston';
 import { Transaction } from '../../database';
 import { DeferredEntity } from '../processing/types';
+import { RefreshIntervalFunction } from '../refresh';
 import {
   DbRefreshStateReferencesRow,
   DbRefreshStateRow,
@@ -48,7 +49,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
     private readonly options: {
       database: Knex;
       logger: Logger;
-      refreshIntervalSeconds: number;
+      refreshInterval: RefreshIntervalFunction;
     },
   ) {}
 
@@ -476,12 +477,10 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
         next_update_at:
           tx.client.config.client === 'sqlite3'
             ? tx.raw(`datetime('now', ?)`, [
-                `${this.options.refreshIntervalSeconds} seconds`,
+                `${this.options.refreshInterval()} seconds`,
               ])
             : tx.raw(
-                `now() + interval '${Number(
-                  this.options.refreshIntervalSeconds,
-                )} seconds'`,
+                `now() + interval '${this.options.refreshInterval()} seconds'`,
               ),
       });
 

plugins/catalog-backend/src/next/index.ts

@@ -18,3 +18,5 @@ export { NextCatalogBuilder } from './NextCatalogBuilder';
 export { createNextRouter } from './NextRouter';
 export * from './processing';
 export * from './stitching';
+export type { RefreshIntervalFunction } from './refresh';
+export { createRandomRefreshInterval } from './refresh';

plugins/catalog-backend/src/next/refresh.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Function that returns the catalog refresh interval in seconds.
+ */
+export type RefreshIntervalFunction = () => number;
+
+/**
+ * Creates a function that returns a random refresh interval between minSeconds and maxSeconds.
+ * @returns {RefreshIntervalFunction} that provides the next refresh interval
+ */
+export function createRandomRefreshInterval(options: {
+  minSeconds: number;
+  maxSeconds: number;
+}): RefreshIntervalFunction {
+  const { minSeconds, maxSeconds } = options;
+  return () => {
+    return Math.random() * (maxSeconds - minSeconds) + minSeconds;
+  };
+}