From 3ca5f700028f036728f426d1835cc81db36fb9c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 25 Nov 2024 15:50:18 +0100 Subject: [PATCH] Ensure that the scheduled worker task doesn't run at an unreasonably high frequency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/weak-meals-doubt.md | 5 +++++ .../src/module/WrapperProviders.ts | 9 +++++++-- .../src/service/IncrementalCatalogBuilder.ts | 9 +++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 .changeset/weak-meals-doubt.md diff --git a/.changeset/weak-meals-doubt.md b/.changeset/weak-meals-doubt.md new file mode 100644 index 0000000000..42199ed296 --- /dev/null +++ b/.changeset/weak-meals-doubt.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-incremental-ingestion': patch +--- + +Ensure that the scheduled worker task doesn't run at an unreasonably high frequency diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts index 738d5733d0..d7be197bf8 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts @@ -119,12 +119,17 @@ export class WrapperProviders { connection, }); - const frequency = Duration.isDuration(burstInterval) + let frequency = Duration.isDuration(burstInterval) ? burstInterval : Duration.fromObject(burstInterval); - const length = Duration.isDuration(burstLength) + if (frequency.as('milliseconds') < 5000) { + frequency = Duration.fromObject({ seconds: 5 }); // don't let it be silly low, to not overload the scheduler + } + + let length = Duration.isDuration(burstLength) ? burstLength : Duration.fromObject(burstLength); + length = length.plus(Duration.fromObject({ minutes: 1 })); // some margin from the actual completion await this.options.scheduler.scheduleTask({ id: provider.getProviderName(), diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts index 04ceef2d26..440e9af9ef 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts @@ -97,12 +97,17 @@ export class IncrementalCatalogBuilder { connection, }); - const frequency = Duration.isDuration(burstInterval) + let frequency = Duration.isDuration(burstInterval) ? burstInterval : Duration.fromObject(burstInterval); - const length = Duration.isDuration(burstLength) + if (frequency.as('milliseconds') < 5000) { + frequency = Duration.fromObject({ seconds: 5 }); // don't let it be silly low, to not overload the scheduler + } + + let length = Duration.isDuration(burstLength) ? burstLength : Duration.fromObject(burstLength); + length = length.plus(Duration.fromObject({ minutes: 1 })); // some margin from the actual completion await scheduler.scheduleTask({ id: provider.getProviderName(),