From 1ef5c2720dff91e898212ee95d967cc39ed94c71 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 22 Sep 2023 15:02:11 +0200 Subject: [PATCH 01/10] feat: turn on some simple monitoring with prometheus for now Signed-off-by: blam --- packages/backend-tasks/package.json | 1 + .../backend-tasks/src/tasks/TaskWorker.ts | 26 ++++++++++- packages/backend-tasks/src/tasks/metrics.ts | 46 +++++++++++++++++++ yarn.lock | 1 + 4 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 packages/backend-tasks/src/tasks/metrics.ts diff --git a/packages/backend-tasks/package.json b/packages/backend-tasks/package.json index f4118b8098..221e1f3a69 100644 --- a/packages/backend-tasks/package.json +++ b/packages/backend-tasks/package.json @@ -41,6 +41,7 @@ "knex": "^2.0.0", "lodash": "^4.17.21", "luxon": "^3.0.0", + "prom-client": "^14.0.1", "uuid": "^8.0.0", "winston": "^3.2.1", "zod": "^3.21.4" diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts index 6e3a88410b..b5c4162f68 100644 --- a/packages/backend-tasks/src/tasks/TaskWorker.ts +++ b/packages/backend-tasks/src/tasks/TaskWorker.ts @@ -18,9 +18,11 @@ import { ConflictError, NotFoundError } from '@backstage/errors'; import { CronTime } from 'cron'; import { Knex } from 'knex'; import { DateTime, Duration } from 'luxon'; +import { Counter, Summary } from 'prom-client'; import { v4 as uuid } from 'uuid'; import { Logger } from 'winston'; import { DbTasksRow, DB_TASKS_TABLE } from '../database/tables'; +import { createCounterMetric, createSummaryMetric } from './metrics'; import { TaskFunction, TaskSettingsV2, taskSettingsV2Schema } from './types'; import { delegateAbortController, nowPlus, sleep } from './util'; @@ -32,13 +34,27 @@ const DEFAULT_WORK_CHECK_FREQUENCY = Duration.fromObject({ seconds: 5 }); * @private */ export class TaskWorker { + private readonly counter: Counter; + private readonly duration: Summary; + constructor( private readonly taskId: string, private readonly fn: TaskFunction, private readonly knex: Knex, private readonly logger: Logger, private readonly workCheckFrequency: Duration = DEFAULT_WORK_CHECK_FREQUENCY, - ) {} + ) { + this.counter = createCounterMetric({ + name: 'backstage_task_runs_total', + help: 'Total number of times a task has been run', + labelNames: ['task_id', 'result'], + }); + this.duration = createSummaryMetric({ + name: 'backstage_task_run_duration_seconds', + help: 'Duration of task runs in seconds', + labelNames: ['task_id', 'result'], + }); + } async start(settings: TaskSettingsV2, options?: { signal?: AbortSignal }) { try { @@ -50,9 +66,13 @@ export class TaskWorker { this.logger.info( `Task worker starting: ${this.taskId}, ${JSON.stringify(settings)}`, ); + this.counter.inc({ task_id: this.taskId, result: 'started' }, 1); + let attemptNum = 1; (async () => { for (;;) { + const endDuration = this.duration.startTimer(); + try { if (settings.initialDelayDuration) { await sleep( @@ -71,6 +91,8 @@ export class TaskWorker { } this.logger.info(`Task worker finished: ${this.taskId}`); + this.counter.inc({ task_id: this.taskId, result: 'completed' }, 1); + endDuration({ task_id: this.taskId, result: 'completed' }); attemptNum = 0; break; } catch (e) { @@ -78,6 +100,8 @@ export class TaskWorker { this.logger.warn( `Task worker failed unexpectedly, attempt number ${attemptNum}, ${e}`, ); + this.counter.inc({ task_id: this.taskId, result: 'failed' }, 1); + endDuration({ task_id: this.taskId, result: 'failed' }); await sleep(Duration.fromObject({ seconds: 1 })); } } diff --git a/packages/backend-tasks/src/tasks/metrics.ts b/packages/backend-tasks/src/tasks/metrics.ts new file mode 100644 index 0000000000..574e64afcb --- /dev/null +++ b/packages/backend-tasks/src/tasks/metrics.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Counter, + CounterConfiguration, + register, + Summary, + SummaryConfiguration, +} from 'prom-client'; + +export function createCounterMetric( + config: CounterConfiguration, +): Counter { + let metric = register.getSingleMetric(config.name); + if (!metric) { + metric = new Counter(config); + register.registerMetric(metric); + } + return metric as Counter; +} + +export function createSummaryMetric( + config: SummaryConfiguration, +): Summary { + let metric = register.getSingleMetric(config.name); + if (!metric) { + metric = new Summary(config); + register.registerMetric(metric); + } + + return metric as Summary; +} diff --git a/yarn.lock b/yarn.lock index de2a8d5ea7..fee4c29a3a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3628,6 +3628,7 @@ __metadata: knex: ^2.0.0 lodash: ^4.17.21 luxon: ^3.0.0 + prom-client: ^14.0.1 uuid: ^8.0.0 wait-for-expect: ^3.0.2 winston: ^3.2.1 From 7184708a2008b7195bbade884ce05343378138c7 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 22 Sep 2023 15:15:31 +0200 Subject: [PATCH 02/10] chore: tinker around a little bit Signed-off-by: blam --- packages/backend-tasks/src/tasks/TaskWorker.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts index b5c4162f68..c83bccb197 100644 --- a/packages/backend-tasks/src/tasks/TaskWorker.ts +++ b/packages/backend-tasks/src/tasks/TaskWorker.ts @@ -71,8 +71,6 @@ export class TaskWorker { let attemptNum = 1; (async () => { for (;;) { - const endDuration = this.duration.startTimer(); - try { if (settings.initialDelayDuration) { await sleep( @@ -82,7 +80,12 @@ export class TaskWorker { } while (!options?.signal?.aborted) { + const endDuration = this.duration.startTimer(); const runResult = await this.runOnce(options?.signal); + endDuration({ task_id: this.taskId, result: 'completed' }); + + this.counter.inc({ task_id: this.taskId, result: 'completed' }, 1); + if (runResult.result === 'abort') { break; } @@ -91,8 +94,7 @@ export class TaskWorker { } this.logger.info(`Task worker finished: ${this.taskId}`); - this.counter.inc({ task_id: this.taskId, result: 'completed' }, 1); - endDuration({ task_id: this.taskId, result: 'completed' }); + attemptNum = 0; break; } catch (e) { @@ -101,7 +103,6 @@ export class TaskWorker { `Task worker failed unexpectedly, attempt number ${attemptNum}, ${e}`, ); this.counter.inc({ task_id: this.taskId, result: 'failed' }, 1); - endDuration({ task_id: this.taskId, result: 'failed' }); await sleep(Duration.fromObject({ seconds: 1 })); } } From 3748620277224dbdd078c7eecae64cf7b8ae994d Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 22 Sep 2023 16:36:20 +0200 Subject: [PATCH 03/10] feat: use OT metrics instead of prom-client Signed-off-by: blam --- packages/backend-tasks/package.json | 2 +- .../backend-tasks/src/tasks/TaskWorker.ts | 32 +++++++++---------- yarn.lock | 2 +- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/packages/backend-tasks/package.json b/packages/backend-tasks/package.json index 221e1f3a69..197caae774 100644 --- a/packages/backend-tasks/package.json +++ b/packages/backend-tasks/package.json @@ -36,12 +36,12 @@ "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/types": "workspace:^", + "@opentelemetry/api": "^1.3.0", "@types/luxon": "^3.0.0", "cron": "^2.0.0", "knex": "^2.0.0", "lodash": "^4.17.21", "luxon": "^3.0.0", - "prom-client": "^14.0.1", "uuid": "^8.0.0", "winston": "^3.2.1", "zod": "^3.21.4" diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts index c83bccb197..a2ed75fef5 100644 --- a/packages/backend-tasks/src/tasks/TaskWorker.ts +++ b/packages/backend-tasks/src/tasks/TaskWorker.ts @@ -18,13 +18,12 @@ import { ConflictError, NotFoundError } from '@backstage/errors'; import { CronTime } from 'cron'; import { Knex } from 'knex'; import { DateTime, Duration } from 'luxon'; -import { Counter, Summary } from 'prom-client'; import { v4 as uuid } from 'uuid'; import { Logger } from 'winston'; import { DbTasksRow, DB_TASKS_TABLE } from '../database/tables'; -import { createCounterMetric, createSummaryMetric } from './metrics'; import { TaskFunction, TaskSettingsV2, taskSettingsV2Schema } from './types'; import { delegateAbortController, nowPlus, sleep } from './util'; +import { metrics, Counter, Histogram } from '@opentelemetry/api'; const DEFAULT_WORK_CHECK_FREQUENCY = Duration.fromObject({ seconds: 5 }); @@ -35,7 +34,7 @@ const DEFAULT_WORK_CHECK_FREQUENCY = Duration.fromObject({ seconds: 5 }); */ export class TaskWorker { private readonly counter: Counter; - private readonly duration: Summary; + private readonly duration: Histogram; constructor( private readonly taskId: string, @@ -44,15 +43,13 @@ export class TaskWorker { private readonly logger: Logger, private readonly workCheckFrequency: Duration = DEFAULT_WORK_CHECK_FREQUENCY, ) { - this.counter = createCounterMetric({ - name: 'backstage_task_runs_total', - help: 'Total number of times a task has been run', - labelNames: ['task_id', 'result'], + const meter = metrics.getMeter('default'); + this.counter = meter.createCounter('backend_task_runs_total', { + description: 'Total number of times a task has been run', }); - this.duration = createSummaryMetric({ - name: 'backstage_task_run_duration_seconds', - help: 'Duration of task runs in seconds', - labelNames: ['task_id', 'result'], + this.duration = meter.createHistogram('backend_task_run_duration', { + description: 'Histogram of task run durations', + unit: 'seconds', }); } @@ -66,7 +63,7 @@ export class TaskWorker { this.logger.info( `Task worker starting: ${this.taskId}, ${JSON.stringify(settings)}`, ); - this.counter.inc({ task_id: this.taskId, result: 'started' }, 1); + this.counter.add(1, { task_id: this.taskId, result: 'started' }); let attemptNum = 1; (async () => { @@ -80,11 +77,13 @@ export class TaskWorker { } while (!options?.signal?.aborted) { - const endDuration = this.duration.startTimer(); + const startTime = process.hrtime(); const runResult = await this.runOnce(options?.signal); - endDuration({ task_id: this.taskId, result: 'completed' }); + const delta = process.hrtime(startTime); + const endTime = delta[0] + delta[1] / 1e9; - this.counter.inc({ task_id: this.taskId, result: 'completed' }, 1); + this.duration.record(endTime, { task_id: this.taskId }); + this.counter.add(1, { task_id: this.taskId, result: 'completed' }); if (runResult.result === 'abort') { break; @@ -94,7 +93,6 @@ export class TaskWorker { } this.logger.info(`Task worker finished: ${this.taskId}`); - attemptNum = 0; break; } catch (e) { @@ -102,7 +100,7 @@ export class TaskWorker { this.logger.warn( `Task worker failed unexpectedly, attempt number ${attemptNum}, ${e}`, ); - this.counter.inc({ task_id: this.taskId, result: 'failed' }, 1); + this.counter.add(1, { task_id: this.taskId, result: 'failed' }); await sleep(Duration.fromObject({ seconds: 1 })); } } diff --git a/yarn.lock b/yarn.lock index fee4c29a3a..79fa81b6f3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3622,13 +3622,13 @@ __metadata: "@backstage/config": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/types": "workspace:^" + "@opentelemetry/api": ^1.3.0 "@types/cron": ^2.0.0 "@types/luxon": ^3.0.0 cron: ^2.0.0 knex: ^2.0.0 lodash: ^4.17.21 luxon: ^3.0.0 - prom-client: ^14.0.1 uuid: ^8.0.0 wait-for-expect: ^3.0.2 winston: ^3.2.1 From 1b85da3f9bcc26c9be29383207bc0d554951def1 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 22 Sep 2023 16:42:02 +0200 Subject: [PATCH 04/10] chore: move metrics collection to a different place to make it simpler Signed-off-by: blam --- packages/backend-tasks/src/tasks/TaskWorker.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts index a2ed75fef5..ecf71c1db4 100644 --- a/packages/backend-tasks/src/tasks/TaskWorker.ts +++ b/packages/backend-tasks/src/tasks/TaskWorker.ts @@ -77,13 +77,7 @@ export class TaskWorker { } while (!options?.signal?.aborted) { - const startTime = process.hrtime(); const runResult = await this.runOnce(options?.signal); - const delta = process.hrtime(startTime); - const endTime = delta[0] + delta[1] / 1e9; - - this.duration.record(endTime, { task_id: this.taskId }); - this.counter.add(1, { task_id: this.taskId, result: 'completed' }); if (runResult.result === 'abort') { break; @@ -100,7 +94,6 @@ export class TaskWorker { this.logger.warn( `Task worker failed unexpectedly, attempt number ${attemptNum}, ${e}`, ); - this.counter.add(1, { task_id: this.taskId, result: 'failed' }); await sleep(Duration.fromObject({ seconds: 1 })); } } @@ -164,10 +157,17 @@ export class TaskWorker { }, Duration.fromISO(taskSettings.timeoutAfterDuration).as('milliseconds')); try { + const startTime = process.hrtime(); await this.fn(taskAbortController.signal); + const delta = process.hrtime(startTime); + const endTime = delta[0] + delta[1] / 1e9; + + this.duration.record(endTime, { task_id: this.taskId }); + this.counter.add(1, { task_id: this.taskId, result: 'completed' }); taskAbortController.abort(); // releases resources } catch (e) { this.logger.error(e); + this.counter.add(1, { task_id: this.taskId, result: 'failed' }); await this.tryReleaseTask(ticket, taskSettings); return { result: 'failed' }; } finally { From 5db102bfdf18719ce8e55a7aae3364925f0337b3 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 22 Sep 2023 16:43:24 +0200 Subject: [PATCH 05/10] feat: instrumentation changeset Signed-off-by: blam --- .changeset/breezy-dryers-thank.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/breezy-dryers-thank.md diff --git a/.changeset/breezy-dryers-thank.md b/.changeset/breezy-dryers-thank.md new file mode 100644 index 0000000000..22be92f530 --- /dev/null +++ b/.changeset/breezy-dryers-thank.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-tasks': patch +--- + +Instrument `backend-tasks` with some counters and histograms for duration From caf288ff034bd9f8de9028772c9893d0025eabb7 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 25 Sep 2023 13:49:37 +0200 Subject: [PATCH 06/10] chore: remove unused metircs Signed-off-by: blam --- packages/backend-tasks/src/tasks/metrics.ts | 46 --------------------- 1 file changed, 46 deletions(-) delete mode 100644 packages/backend-tasks/src/tasks/metrics.ts diff --git a/packages/backend-tasks/src/tasks/metrics.ts b/packages/backend-tasks/src/tasks/metrics.ts deleted file mode 100644 index 574e64afcb..0000000000 --- a/packages/backend-tasks/src/tasks/metrics.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - Counter, - CounterConfiguration, - register, - Summary, - SummaryConfiguration, -} from 'prom-client'; - -export function createCounterMetric( - config: CounterConfiguration, -): Counter { - let metric = register.getSingleMetric(config.name); - if (!metric) { - metric = new Counter(config); - register.registerMetric(metric); - } - return metric as Counter; -} - -export function createSummaryMetric( - config: SummaryConfiguration, -): Summary { - let metric = register.getSingleMetric(config.name); - if (!metric) { - metric = new Summary(config); - register.registerMetric(metric); - } - - return metric as Summary; -} From 6916c4b291b0fd0cec26bb4b3f1154089c00240b Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Sep 2023 14:54:40 +0200 Subject: [PATCH 07/10] feat: move the implementation to the scheduler instead Signed-off-by: blam --- .../src/tasks/PluginTaskSchedulerImpl.ts | 44 +++++++++++++++++-- .../backend-tasks/src/tasks/TaskWorker.ts | 23 +--------- 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts index b9ac80cb47..83acc9e972 100644 --- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts +++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts @@ -28,7 +28,8 @@ import { TaskSettingsV2, } from './types'; import { validateId } from './util'; - +import { TaskFunction } from './types'; +import { metrics, Counter, Histogram } from '@opentelemetry/api'; /** * Implements the actual task management. */ @@ -36,10 +37,22 @@ export class PluginTaskSchedulerImpl implements PluginTaskScheduler { private readonly localTasksById = new Map(); private readonly allScheduledTasks: TaskDescriptor[] = []; + private readonly counter: Counter; + private readonly duration: Histogram; + constructor( private readonly databaseFactory: () => Promise, private readonly logger: Logger, - ) {} + ) { + const meter = metrics.getMeter('default'); + this.counter = meter.createCounter('backend_task_runs_total', { + description: 'Total number of times a task has been run', + }); + this.duration = meter.createHistogram('backend_task_run_duration', { + description: 'Histogram of task run durations', + unit: 'seconds', + }); + } async triggerTask(id: string): Promise { const localTask = this.localTasksById.get(id); @@ -70,7 +83,7 @@ export class PluginTaskSchedulerImpl implements PluginTaskScheduler { const knex = await this.databaseFactory(); const worker = new TaskWorker( task.id, - task.fn, + this.wrapInMetrics(task.fn, { labels: { taskId: task.id, scope } }), knex, this.logger.child({ task: task.id }), ); @@ -78,7 +91,7 @@ export class PluginTaskSchedulerImpl implements PluginTaskScheduler { } else { const worker = new LocalTaskWorker( task.id, - task.fn, + this.wrapInMetrics(task.fn, { labels: { taskId: task.id, scope } }), this.logger.child({ task: task.id }), ); worker.start(settings, { signal: task.signal }); @@ -103,6 +116,29 @@ export class PluginTaskSchedulerImpl implements PluginTaskScheduler { async getScheduledTasks(): Promise { return this.allScheduledTasks; } + + private wrapInMetrics( + fn: TaskFunction, + opts: { labels: Record }, + ): TaskFunction { + return async abort => { + this.counter.add(1, { ...opts.labels, result: 'started' }); + + const startTime = process.hrtime(); + + try { + await fn(abort); + this.counter.add(1, { ...opts.labels, result: 'completed' }); + } catch (ex) { + this.counter.add(1, { ...opts.labels, result: 'failed' }); + throw ex; + } finally { + const delta = process.hrtime(startTime); + const endTime = delta[0] + delta[1] / 1e9; + this.duration.record(endTime, { ...opts.labels }); + } + }; + } } export function parseDuration( diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts index ecf71c1db4..4e10f0b5b2 100644 --- a/packages/backend-tasks/src/tasks/TaskWorker.ts +++ b/packages/backend-tasks/src/tasks/TaskWorker.ts @@ -23,7 +23,6 @@ import { Logger } from 'winston'; import { DbTasksRow, DB_TASKS_TABLE } from '../database/tables'; import { TaskFunction, TaskSettingsV2, taskSettingsV2Schema } from './types'; import { delegateAbortController, nowPlus, sleep } from './util'; -import { metrics, Counter, Histogram } from '@opentelemetry/api'; const DEFAULT_WORK_CHECK_FREQUENCY = Duration.fromObject({ seconds: 5 }); @@ -33,25 +32,13 @@ const DEFAULT_WORK_CHECK_FREQUENCY = Duration.fromObject({ seconds: 5 }); * @private */ export class TaskWorker { - private readonly counter: Counter; - private readonly duration: Histogram; - constructor( private readonly taskId: string, private readonly fn: TaskFunction, private readonly knex: Knex, private readonly logger: Logger, private readonly workCheckFrequency: Duration = DEFAULT_WORK_CHECK_FREQUENCY, - ) { - const meter = metrics.getMeter('default'); - this.counter = meter.createCounter('backend_task_runs_total', { - description: 'Total number of times a task has been run', - }); - this.duration = meter.createHistogram('backend_task_run_duration', { - description: 'Histogram of task run durations', - unit: 'seconds', - }); - } + ) {} async start(settings: TaskSettingsV2, options?: { signal?: AbortSignal }) { try { @@ -63,7 +50,6 @@ export class TaskWorker { this.logger.info( `Task worker starting: ${this.taskId}, ${JSON.stringify(settings)}`, ); - this.counter.add(1, { task_id: this.taskId, result: 'started' }); let attemptNum = 1; (async () => { @@ -157,17 +143,10 @@ export class TaskWorker { }, Duration.fromISO(taskSettings.timeoutAfterDuration).as('milliseconds')); try { - const startTime = process.hrtime(); await this.fn(taskAbortController.signal); - const delta = process.hrtime(startTime); - const endTime = delta[0] + delta[1] / 1e9; - - this.duration.record(endTime, { task_id: this.taskId }); - this.counter.add(1, { task_id: this.taskId, result: 'completed' }); taskAbortController.abort(); // releases resources } catch (e) { this.logger.error(e); - this.counter.add(1, { task_id: this.taskId, result: 'failed' }); await this.tryReleaseTask(ticket, taskSettings); return { result: 'failed' }; } finally { From 0dada05a787c1f74774920f125fe7e084f5c38eb Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 2 Oct 2023 14:06:50 +0200 Subject: [PATCH 08/10] chore: fixing changeset Signed-off-by: blam --- .changeset/breezy-dryers-thank.md | 7 ++++++- .../src/tasks/PluginTaskSchedulerImpl.ts | 13 +++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.changeset/breezy-dryers-thank.md b/.changeset/breezy-dryers-thank.md index 22be92f530..317d2316b1 100644 --- a/.changeset/breezy-dryers-thank.md +++ b/.changeset/breezy-dryers-thank.md @@ -2,4 +2,9 @@ '@backstage/backend-tasks': patch --- -Instrument `backend-tasks` with some counters and histograms for duration +Instrument `backend-tasks` with some counters and histograms for duration. + +`backend_task_runs_total`: Counter with the total number of times a task has been run. +`backend_task_run_duration`: Histogram with the run durations for each task. + +Both these metrics have come with `result` `taskId` and `scope` labels for finer grained grouping andd diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts index 83acc9e972..9971d47ab6 100644 --- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts +++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts @@ -30,6 +30,7 @@ import { import { validateId } from './util'; import { TaskFunction } from './types'; import { metrics, Counter, Histogram } from '@opentelemetry/api'; + /** * Implements the actual task management. */ @@ -122,20 +123,24 @@ export class PluginTaskSchedulerImpl implements PluginTaskScheduler { opts: { labels: Record }, ): TaskFunction { return async abort => { - this.counter.add(1, { ...opts.labels, result: 'started' }); + const labels = { + ...opts.labels, + }; + this.counter.add(1, { ...labels, result: 'started' }); const startTime = process.hrtime(); try { await fn(abort); - this.counter.add(1, { ...opts.labels, result: 'completed' }); + labels.result = 'completed'; } catch (ex) { - this.counter.add(1, { ...opts.labels, result: 'failed' }); + labels.result = 'failed'; throw ex; } finally { const delta = process.hrtime(startTime); const endTime = delta[0] + delta[1] / 1e9; - this.duration.record(endTime, { ...opts.labels }); + this.counter.add(1, labels); + this.duration.record(endTime, labels); } }; } From bbbd853875c614c138da8a260eed0e5bda027d64 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 3 Oct 2023 10:36:26 +0200 Subject: [PATCH 09/10] chore: update the metric names Signed-off-by: blam --- packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts index 9971d47ab6..38561c7a8c 100644 --- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts +++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts @@ -46,10 +46,10 @@ export class PluginTaskSchedulerImpl implements PluginTaskScheduler { private readonly logger: Logger, ) { const meter = metrics.getMeter('default'); - this.counter = meter.createCounter('backend_task_runs_total', { + this.counter = meter.createCounter('backend_tasks.task.runs.count', { description: 'Total number of times a task has been run', }); - this.duration = meter.createHistogram('backend_task_run_duration', { + this.duration = meter.createHistogram('backend_tasks.task.runs.duration', { description: 'Histogram of task run durations', unit: 'seconds', }); From e7d9d346f4bc90d05bfe5ca6953de7cf1fc42c82 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 3 Oct 2023 10:37:32 +0200 Subject: [PATCH 10/10] chore: fixing changeset Signed-off-by: blam --- .changeset/breezy-dryers-thank.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.changeset/breezy-dryers-thank.md b/.changeset/breezy-dryers-thank.md index 317d2316b1..7a742db1bf 100644 --- a/.changeset/breezy-dryers-thank.md +++ b/.changeset/breezy-dryers-thank.md @@ -4,7 +4,7 @@ Instrument `backend-tasks` with some counters and histograms for duration. -`backend_task_runs_total`: Counter with the total number of times a task has been run. -`backend_task_run_duration`: Histogram with the run durations for each task. +`backend_tasks.task.runs.count`: Counter with the total number of times a task has been run. +`backend_tasks.task.runs.duration`: Histogram with the run durations for each task. -Both these metrics have come with `result` `taskId` and `scope` labels for finer grained grouping andd +Both these metrics have come with `result` `taskId` and `scope` labels for finer grained grouping.