Ensure that the scheduled worker task doesn't run at an unreasonably high frequency

Signed-off-by: Fredrik Adelöw <freben@gmail.com>

.changeset/weak-meals-doubt.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-incremental-ingestion': patch
+---
+
+Ensure that the scheduled worker task doesn't run at an unreasonably high frequency

plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts

@@ -119,12 +119,17 @@ export class WrapperProviders {
         connection,
       });
 
-      const frequency = Duration.isDuration(burstInterval)
+      let frequency = Duration.isDuration(burstInterval)
         ? burstInterval
         : Duration.fromObject(burstInterval);
-      const length = Duration.isDuration(burstLength)
+      if (frequency.as('milliseconds') < 5000) {
+        frequency = Duration.fromObject({ seconds: 5 }); // don't let it be silly low, to not overload the scheduler
+      }
+
+      let length = Duration.isDuration(burstLength)
         ? burstLength
         : Duration.fromObject(burstLength);
+      length = length.plus(Duration.fromObject({ minutes: 1 })); // some margin from the actual completion
 
       await this.options.scheduler.scheduleTask({
         id: provider.getProviderName(),

plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts

@@ -97,12 +97,17 @@ export class IncrementalCatalogBuilder {
           connection,
         });
 
-        const frequency = Duration.isDuration(burstInterval)
+        let frequency = Duration.isDuration(burstInterval)
           ? burstInterval
           : Duration.fromObject(burstInterval);
-        const length = Duration.isDuration(burstLength)
+        if (frequency.as('milliseconds') < 5000) {
+          frequency = Duration.fromObject({ seconds: 5 }); // don't let it be silly low, to not overload the scheduler
+        }
+
+        let length = Duration.isDuration(burstLength)
           ? burstLength
           : Duration.fromObject(burstLength);
+        length = length.plus(Duration.fromObject({ minutes: 1 })); // some margin from the actual completion
 
         await scheduler.scheduleTask({
           id: provider.getProviderName(),