diff --git a/.changeset/breezy-dryers-thank.md b/.changeset/breezy-dryers-thank.md new file mode 100644 index 0000000000..7a742db1bf --- /dev/null +++ b/.changeset/breezy-dryers-thank.md @@ -0,0 +1,10 @@ +--- +'@backstage/backend-tasks': patch +--- + +Instrument `backend-tasks` with some counters and histograms for duration. + +`backend_tasks.task.runs.count`: Counter with the total number of times a task has been run. +`backend_tasks.task.runs.duration`: Histogram with the run durations for each task. + +Both these metrics have come with `result` `taskId` and `scope` labels for finer grained grouping. diff --git a/packages/backend-tasks/package.json b/packages/backend-tasks/package.json index f4118b8098..197caae774 100644 --- a/packages/backend-tasks/package.json +++ b/packages/backend-tasks/package.json @@ -36,6 +36,7 @@ "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/types": "workspace:^", + "@opentelemetry/api": "^1.3.0", "@types/luxon": "^3.0.0", "cron": "^2.0.0", "knex": "^2.0.0", diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts index b9ac80cb47..38561c7a8c 100644 --- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts +++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts @@ -28,6 +28,8 @@ import { TaskSettingsV2, } from './types'; import { validateId } from './util'; +import { TaskFunction } from './types'; +import { metrics, Counter, Histogram } from '@opentelemetry/api'; /** * Implements the actual task management. @@ -36,10 +38,22 @@ export class PluginTaskSchedulerImpl implements PluginTaskScheduler { private readonly localTasksById = new Map(); private readonly allScheduledTasks: TaskDescriptor[] = []; + private readonly counter: Counter; + private readonly duration: Histogram; + constructor( private readonly databaseFactory: () => Promise, private readonly logger: Logger, - ) {} + ) { + const meter = metrics.getMeter('default'); + this.counter = meter.createCounter('backend_tasks.task.runs.count', { + description: 'Total number of times a task has been run', + }); + this.duration = meter.createHistogram('backend_tasks.task.runs.duration', { + description: 'Histogram of task run durations', + unit: 'seconds', + }); + } async triggerTask(id: string): Promise { const localTask = this.localTasksById.get(id); @@ -70,7 +84,7 @@ export class PluginTaskSchedulerImpl implements PluginTaskScheduler { const knex = await this.databaseFactory(); const worker = new TaskWorker( task.id, - task.fn, + this.wrapInMetrics(task.fn, { labels: { taskId: task.id, scope } }), knex, this.logger.child({ task: task.id }), ); @@ -78,7 +92,7 @@ export class PluginTaskSchedulerImpl implements PluginTaskScheduler { } else { const worker = new LocalTaskWorker( task.id, - task.fn, + this.wrapInMetrics(task.fn, { labels: { taskId: task.id, scope } }), this.logger.child({ task: task.id }), ); worker.start(settings, { signal: task.signal }); @@ -103,6 +117,33 @@ export class PluginTaskSchedulerImpl implements PluginTaskScheduler { async getScheduledTasks(): Promise { return this.allScheduledTasks; } + + private wrapInMetrics( + fn: TaskFunction, + opts: { labels: Record }, + ): TaskFunction { + return async abort => { + const labels = { + ...opts.labels, + }; + this.counter.add(1, { ...labels, result: 'started' }); + + const startTime = process.hrtime(); + + try { + await fn(abort); + labels.result = 'completed'; + } catch (ex) { + labels.result = 'failed'; + throw ex; + } finally { + const delta = process.hrtime(startTime); + const endTime = delta[0] + delta[1] / 1e9; + this.counter.add(1, labels); + this.duration.record(endTime, labels); + } + }; + } } export function parseDuration( diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts index 6e3a88410b..4e10f0b5b2 100644 --- a/packages/backend-tasks/src/tasks/TaskWorker.ts +++ b/packages/backend-tasks/src/tasks/TaskWorker.ts @@ -50,6 +50,7 @@ export class TaskWorker { this.logger.info( `Task worker starting: ${this.taskId}, ${JSON.stringify(settings)}`, ); + let attemptNum = 1; (async () => { for (;;) { @@ -63,6 +64,7 @@ export class TaskWorker { while (!options?.signal?.aborted) { const runResult = await this.runOnce(options?.signal); + if (runResult.result === 'abort') { break; } diff --git a/yarn.lock b/yarn.lock index de2a8d5ea7..79fa81b6f3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3622,6 +3622,7 @@ __metadata: "@backstage/config": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/types": "workspace:^" + "@opentelemetry/api": ^1.3.0 "@types/cron": ^2.0.0 "@types/luxon": ^3.0.0 cron: ^2.0.0