From 633506ef06e1c25609a32d38080a303b5f8a7765 Mon Sep 17 00:00:00 2001 From: blakeromano-il Date: Mon, 16 May 2022 12:06:58 -0400 Subject: [PATCH 1/6] add restart to task workers Signed-off-by: blakeromano-il --- packages/backend-tasks/src/tasks/LocalTaskWorker.ts | 1 + packages/backend-tasks/src/tasks/TaskWorker.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/backend-tasks/src/tasks/LocalTaskWorker.ts b/packages/backend-tasks/src/tasks/LocalTaskWorker.ts index 89f314a1ec..00765d7db0 100644 --- a/packages/backend-tasks/src/tasks/LocalTaskWorker.ts +++ b/packages/backend-tasks/src/tasks/LocalTaskWorker.ts @@ -64,6 +64,7 @@ export class LocalTaskWorker { this.logger.info(`Task worker finished: ${this.taskId}`); } catch (e) { this.logger.warn(`Task worker failed unexpectedly, ${e}`); + this.start(settings, options) } })(); } diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts index d2269d8118..ba788b23ad 100644 --- a/packages/backend-tasks/src/tasks/TaskWorker.ts +++ b/packages/backend-tasks/src/tasks/TaskWorker.ts @@ -72,6 +72,7 @@ export class TaskWorker { this.logger.info(`Task worker finished: ${this.taskId}`); } catch (e) { + this.start(settings, options) this.logger.warn(`Task worker failed unexpectedly, ${e}`); } })(); From 07f65da22ae2489ea8bd9319b3a5719c21d98208 Mon Sep 17 00:00:00 2001 From: blakeromano-il Date: Mon, 16 May 2022 12:09:24 -0400 Subject: [PATCH 2/6] change to after warn for both Signed-off-by: blakeromano-il --- packages/backend-tasks/src/tasks/TaskWorker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts index ba788b23ad..c54dc0a857 100644 --- a/packages/backend-tasks/src/tasks/TaskWorker.ts +++ b/packages/backend-tasks/src/tasks/TaskWorker.ts @@ -72,8 +72,8 @@ export class TaskWorker { this.logger.info(`Task worker finished: ${this.taskId}`); } catch (e) { - this.start(settings, options) this.logger.warn(`Task worker failed unexpectedly, ${e}`); + this.start(settings, options) } })(); } From d3afb6983a9e886bb548e786314b4b9a1e41ccb9 Mon Sep 17 00:00:00 2001 From: blakeromano-il Date: Mon, 16 May 2022 12:54:10 -0400 Subject: [PATCH 3/6] convert to loop Signed-off-by: blakeromano-il --- .../src/tasks/LocalTaskWorker.ts | 49 ++++++++++--------- .../backend-tasks/src/tasks/TaskWorker.ts | 44 ++++++++++------- 2 files changed, 52 insertions(+), 41 deletions(-) diff --git a/packages/backend-tasks/src/tasks/LocalTaskWorker.ts b/packages/backend-tasks/src/tasks/LocalTaskWorker.ts index 00765d7db0..c7ed759316 100644 --- a/packages/backend-tasks/src/tasks/LocalTaskWorker.ts +++ b/packages/backend-tasks/src/tasks/LocalTaskWorker.ts @@ -40,31 +40,36 @@ export class LocalTaskWorker { this.logger.info( `Task worker starting: ${this.taskId}, ${JSON.stringify(settings)}`, ); - + const success = false; + let attemptNum = 1; (async () => { - try { - if (settings.initialDelayDuration) { - await this.sleep( - Duration.fromISO(settings.initialDelayDuration), - options?.signal, + while (!success) { + try { + if (settings.initialDelayDuration) { + await this.sleep( + Duration.fromISO(settings.initialDelayDuration), + options?.signal, + ); + } + + while (!options?.signal?.aborted) { + const startTime = process.hrtime(); + await this.runOnce(settings, options?.signal); + const timeTaken = process.hrtime(startTime); + await this.waitUntilNext( + settings, + (timeTaken[0] + timeTaken[1] / 1e9) * 1000, + options?.signal, + ); + } + this.logger.info(`Task worker finished: ${this.taskId}`); + break; + } catch (e) { + attemptNum += 1; + this.logger.warn( + `Task worker failed unexpectedly, attempt number ${attemptNum}, ${e}`, ); } - - while (!options?.signal?.aborted) { - const startTime = process.hrtime(); - await this.runOnce(settings, options?.signal); - const timeTaken = process.hrtime(startTime); - await this.waitUntilNext( - settings, - (timeTaken[0] + timeTaken[1] / 1e9) * 1000, - options?.signal, - ); - } - - this.logger.info(`Task worker finished: ${this.taskId}`); - } catch (e) { - this.logger.warn(`Task worker failed unexpectedly, ${e}`); - this.start(settings, options) } })(); } diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts index c54dc0a857..e74b8b9cdb 100644 --- a/packages/backend-tasks/src/tasks/TaskWorker.ts +++ b/packages/backend-tasks/src/tasks/TaskWorker.ts @@ -51,29 +51,35 @@ export class TaskWorker { this.logger.info( `Task worker starting: ${this.taskId}, ${JSON.stringify(settings)}`, ); - + const success = false; + let attemptNum = 1; (async () => { - try { - if (settings.initialDelayDuration) { - await sleep( - Duration.fromISO(settings.initialDelayDuration), - options?.signal, - ); - } - - while (!options?.signal?.aborted) { - const runResult = await this.runOnce(options?.signal); - if (runResult.result === 'abort') { - break; + while (!success) { + try { + if (settings.initialDelayDuration) { + await sleep( + Duration.fromISO(settings.initialDelayDuration), + options?.signal, + ); } - await sleep(this.workCheckFrequency, options?.signal); - } + while (!options?.signal?.aborted) { + const runResult = await this.runOnce(options?.signal); + if (runResult.result === 'abort') { + break; + } - this.logger.info(`Task worker finished: ${this.taskId}`); - } catch (e) { - this.logger.warn(`Task worker failed unexpectedly, ${e}`); - this.start(settings, options) + await sleep(this.workCheckFrequency, options?.signal); + } + + this.logger.info(`Task worker finished: ${this.taskId}`); + break; + } catch (e) { + attemptNum += 1; + this.logger.warn( + `Task worker failed unexpectedly, attempt number ${attemptNum}, ${e}`, + ); + } } })(); } From fde10d24f66d23fa8e02688ba3e01b91e265ffe4 Mon Sep 17 00:00:00 2001 From: blakeromano-il Date: Wed, 18 May 2022 09:00:34 -0400 Subject: [PATCH 4/6] changeset Signed-off-by: blakeromano-il --- .changeset/funny-suns-pay.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/funny-suns-pay.md diff --git a/.changeset/funny-suns-pay.md b/.changeset/funny-suns-pay.md new file mode 100644 index 0000000000..b736d34cc1 --- /dev/null +++ b/.changeset/funny-suns-pay.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-tasks': patch +--- + +Allow tasks that fail to retry on a loop emitting a warning log every time it fails with the amount of attempts it has From a9f6b49f1c1f2b0f2fb661f78547cefb14e7d952 Mon Sep 17 00:00:00 2001 From: blakeromano-il Date: Wed, 18 May 2022 09:09:22 -0400 Subject: [PATCH 5/6] reset counter, for loop, sleep 1s Signed-off-by: blakeromano-il --- packages/backend-tasks/src/tasks/LocalTaskWorker.ts | 5 +++-- packages/backend-tasks/src/tasks/TaskWorker.ts | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/backend-tasks/src/tasks/LocalTaskWorker.ts b/packages/backend-tasks/src/tasks/LocalTaskWorker.ts index c7ed759316..1326a6cd9d 100644 --- a/packages/backend-tasks/src/tasks/LocalTaskWorker.ts +++ b/packages/backend-tasks/src/tasks/LocalTaskWorker.ts @@ -40,10 +40,9 @@ export class LocalTaskWorker { this.logger.info( `Task worker starting: ${this.taskId}, ${JSON.stringify(settings)}`, ); - const success = false; let attemptNum = 1; (async () => { - while (!success) { + for (;;) { try { if (settings.initialDelayDuration) { await this.sleep( @@ -63,12 +62,14 @@ export class LocalTaskWorker { ); } this.logger.info(`Task worker finished: ${this.taskId}`); + attemptNum = 0; break; } catch (e) { attemptNum += 1; this.logger.warn( `Task worker failed unexpectedly, attempt number ${attemptNum}, ${e}`, ); + await sleep(Duration.fromISO('P1S')); } } })(); diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts index e74b8b9cdb..c19dedc117 100644 --- a/packages/backend-tasks/src/tasks/TaskWorker.ts +++ b/packages/backend-tasks/src/tasks/TaskWorker.ts @@ -51,10 +51,9 @@ export class TaskWorker { this.logger.info( `Task worker starting: ${this.taskId}, ${JSON.stringify(settings)}`, ); - const success = false; let attemptNum = 1; (async () => { - while (!success) { + for (;;) { try { if (settings.initialDelayDuration) { await sleep( @@ -73,12 +72,14 @@ export class TaskWorker { } this.logger.info(`Task worker finished: ${this.taskId}`); + attemptNum = 0; break; } catch (e) { attemptNum += 1; this.logger.warn( `Task worker failed unexpectedly, attempt number ${attemptNum}, ${e}`, ); + await sleep(Duration.fromISO('P1S')); } } })(); From 87d438849396d679915fec7c3f6cd080b7560127 Mon Sep 17 00:00:00 2001 From: blakeromano-il Date: Wed, 18 May 2022 11:53:39 -0400 Subject: [PATCH 6/6] change duration.fromObject Signed-off-by: blakeromano-il --- packages/backend-tasks/src/tasks/LocalTaskWorker.ts | 2 +- packages/backend-tasks/src/tasks/TaskWorker.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-tasks/src/tasks/LocalTaskWorker.ts b/packages/backend-tasks/src/tasks/LocalTaskWorker.ts index 1326a6cd9d..b7dd7c96c1 100644 --- a/packages/backend-tasks/src/tasks/LocalTaskWorker.ts +++ b/packages/backend-tasks/src/tasks/LocalTaskWorker.ts @@ -69,7 +69,7 @@ export class LocalTaskWorker { this.logger.warn( `Task worker failed unexpectedly, attempt number ${attemptNum}, ${e}`, ); - await sleep(Duration.fromISO('P1S')); + await sleep(Duration.fromObject({ seconds: 1 })); } } })(); diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts index c19dedc117..07e1cd90ec 100644 --- a/packages/backend-tasks/src/tasks/TaskWorker.ts +++ b/packages/backend-tasks/src/tasks/TaskWorker.ts @@ -79,7 +79,7 @@ export class TaskWorker { this.logger.warn( `Task worker failed unexpectedly, attempt number ${attemptNum}, ${e}`, ); - await sleep(Duration.fromISO('P1S')); + await sleep(Duration.fromObject({ seconds: 1 })); } } })();