diff --git a/.changeset/flat-items-perform.md b/.changeset/flat-items-perform.md new file mode 100644 index 0000000000..4bc846ba7e --- /dev/null +++ b/.changeset/flat-items-perform.md @@ -0,0 +1,38 @@ +--- +'@backstage/plugin-scaffolder-backend': minor +--- + +Added a set of default Prometheus metrics around scaffolding. See below for a list of metrics and an explanation of their labels: + +- `scaffolder_task_count`: Tracks successful task runs. + + Labels: + + - `template`: The entity ref of the scaffolded template + - `user`: The entity ref of the user that invoked the template run + - `result`: A string describing whether the task ran successfully, failed, or was skipped + +- `scaffolder_task_duration`: a histogram which tracks the duration of a task run + + Labels: + + - `template`: The entity ref of the scaffolded template + - `result`: A boolean describing whether the task ran successfully + +- `scaffolder_step_count`: a count that tracks each step run + + Labels: + + - `template`: The entity ref of the scaffolded template + - `step`: The name of the step that was run + - `result`: A string describing whether the task ran successfully, failed, or was skipped + +- `scaffolder_step_duration`: a histogram which tracks the duration of each step run + + Labels: + + - `template`: The entity ref of the scaffolded template + - `step`: The name of the step that was run + - `result`: A string describing whether the task ran successfully, failed, or was skipped + +You can find a guide for running Prometheus metrics here: https://github.com/backstage/backstage/blob/master/contrib/docs/tutorials/prometheus-metrics.md diff --git a/contrib/docs/tutorials/prometheus-metrics.md b/contrib/docs/tutorials/prometheus-metrics.md index 5b7dc61df3..56989e709c 100644 --- a/contrib/docs/tutorials/prometheus-metrics.md +++ b/contrib/docs/tutorials/prometheus-metrics.md @@ -106,3 +106,7 @@ There are some custom metrics that have been added to Backstage will be output f - `catalog_processing_duration_seconds`: Time spent executing the full processing flow - `catalog_processors_duration_seconds`: Time spent executing catalog processors - `catalog_processing_queue_delay_seconds`: The amount of delay between being scheduled for processing, and the start of actually being processed +- `scaffolder_task_count`: Tracks successful task runs. +- `scaffolder_task_duration`: a histogram which tracks the duration of a task run +- `scaffolder_step_count`: a count that tracks each step run +- `scaffolder_step_duration`: a histogram which tracks the duration of each step run diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index a21423f865..ab3b24dcdd 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -72,6 +72,7 @@ "octokit": "^2.0.0", "octokit-plugin-create-pull-request": "^3.10.0", "p-limit": "^3.1.0", + "prom-client": "^14.0.1", "uuid": "^8.2.0", "vm2": "^3.9.11", "winston": "^3.2.1", diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index c434d9802b..5893a88485 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -26,7 +26,7 @@ import { PassThrough } from 'stream'; import { generateExampleOutput, isTruthy } from './helper'; import { validate as validateJsonSchema } from 'jsonschema'; import { parseRepoUrl } from '../actions/builtin/publish/util'; -import { TemplateActionRegistry } from '../actions'; +import { TemplateAction, TemplateActionRegistry } from '../actions'; import { TemplateFilter, SecureTemplater, @@ -39,6 +39,7 @@ import { TaskStep, } from '@backstage/plugin-scaffolder-common'; import { UserEntity } from '@backstage/catalog-model'; +import { createCounterMetric, createHistogramMetric } from '../../util/metrics'; type NunjucksWorkflowRunnerOptions = { workingDirectory: string; @@ -97,6 +98,7 @@ const createStepLogger = ({ export class NunjucksWorkflowRunner implements WorkflowRunner { constructor(private readonly options: NunjucksWorkflowRunnerOptions) {} + private readonly tracker = scaffoldingTracker(); private isSingleTemplateString(input: string) { const { parser, nodes } = nunjucks as unknown as { @@ -202,10 +204,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { }); try { + const taskTrack = await this.tracker.taskStart(task); await fs.ensureDir(workspacePath); - await task.emitLog( - `Starting up task with ${task.spec.steps.length} steps`, - ); const context: TemplateContext = { parameters: task.spec.parameters, @@ -214,6 +214,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { }; for (const step of task.spec.steps) { + const stepTrack = await this.tracker.stepStart(task, step); try { if (step.if) { const ifResult = await this.render( @@ -222,19 +223,11 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { renderTemplate, ); if (!isTruthy(ifResult)) { - await task.emitLog( - `Skipping step ${step.id} because it's if condition was false`, - { stepId: step.id, status: 'skipped' }, - ); + await stepTrack.skipFalsy(); continue; } } - await task.emitLog(`Beginning step ${step.name}`, { - stepId: step.id, - status: 'processing', - }); - const action = this.options.actionRegistry.get(step.action); const { taskLogger, streamLogger } = createStepLogger({ task, step }); @@ -266,13 +259,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { )}`, ); if (!action.supportsDryRun) { - task.emitLog( - `Skipping because ${action.id} does not support dry-run`, - { - stepId: step.id, - status: 'skipped', - }, - ); + await taskTrack.skipDryRun(step, action); const outputSchema = action.schema?.output; if (outputSchema) { context.steps[step.id] = { @@ -341,20 +328,16 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { context.steps[step.id] = { output: stepOutput }; - await task.emitLog(`Finished step ${step.name}`, { - stepId: step.id, - status: 'completed', - }); + await stepTrack.markSuccessful(); } catch (err) { - await task.emitLog(String(err.stack), { - stepId: step.id, - status: 'failed', - }); + await taskTrack.markFailed(step, err); + await stepTrack.markFailed(); throw err; } } const output = this.render(task.spec.output, context, renderTemplate); + await taskTrack.markSuccessful(); return { output }; } finally { @@ -364,3 +347,128 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { } } } + +function scaffoldingTracker() { + const taskCount = createCounterMetric({ + name: 'scaffolder_task_count', + help: 'Count of task runs', + labelNames: ['template', 'user', 'result'], + }); + const taskDuration = createHistogramMetric({ + name: 'scaffolder_task_duration', + help: 'Duration of a task run', + labelNames: ['template', 'result'], + }); + const stepCount = createCounterMetric({ + name: 'scaffolder_step_count', + help: 'Count of step runs', + labelNames: ['template', 'step', 'result'], + }); + const stepDuration = createHistogramMetric({ + name: 'scaffolder_step_duration', + help: 'Duration of a step runs', + labelNames: ['template', 'step', 'result'], + }); + + async function taskStart(task: TaskContext) { + await task.emitLog(`Starting up task with ${task.spec.steps.length} steps`); + const template = task.spec.templateInfo?.entityRef || ''; + const user = task.spec.user?.ref || ''; + + const taskTimer = taskDuration.startTimer({ + template, + }); + + async function skipDryRun( + step: TaskStep, + action: TemplateAction, + ) { + task.emitLog(`Skipping because ${action.id} does not support dry-run`, { + stepId: step.id, + status: 'skipped', + }); + } + + async function markSuccessful() { + taskCount.inc({ + template, + user, + result: 'ok', + }); + taskTimer({ result: 'ok' }); + } + + async function markFailed(step: TaskStep, err: Error) { + await task.emitLog(String(err.stack), { + stepId: step.id, + status: 'failed', + }); + taskCount.inc({ + template, + user, + result: 'failed', + }); + taskTimer({ result: 'failed' }); + } + + return { + skipDryRun, + markSuccessful, + markFailed, + }; + } + + async function stepStart(task: TaskContext, step: TaskStep) { + await task.emitLog(`Beginning step ${step.name}`, { + stepId: step.id, + status: 'processing', + }); + const template = task.spec.templateInfo?.entityRef || ''; + + const stepTimer = stepDuration.startTimer({ + template, + step: step.name, + }); + + async function markSuccessful() { + await task.emitLog(`Finished step ${step.name}`, { + stepId: step.id, + status: 'completed', + }); + stepCount.inc({ + template, + step: step.name, + result: 'ok', + }); + stepTimer({ result: 'ok' }); + } + + async function markFailed() { + stepCount.inc({ + template, + step: step.name, + result: 'failed', + }); + stepTimer({ result: 'failed' }); + } + + async function skipFalsy() { + await task.emitLog( + `Skipping step ${step.id} because its if condition was false`, + { stepId: step.id, status: 'skipped' }, + ); + stepTimer({ result: 'skipped' }); + } + + return { + markSuccessful, + markFailed, + skipFalsy, + }; + } + + return { + taskStart, + stepStart, + }; +} diff --git a/plugins/scaffolder-backend/src/util/metrics.ts b/plugins/scaffolder-backend/src/util/metrics.ts new file mode 100644 index 0000000000..207135f234 --- /dev/null +++ b/plugins/scaffolder-backend/src/util/metrics.ts @@ -0,0 +1,73 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Counter, + CounterConfiguration, + Gauge, + GaugeConfiguration, + Histogram, + HistogramConfiguration, + register, + Summary, + SummaryConfiguration, +} from 'prom-client'; + +export function createCounterMetric( + config: CounterConfiguration, +): Counter { + let metric = register.getSingleMetric(config.name); + if (!metric) { + metric = new Counter(config); + register.registerMetric(metric); + } + return metric as Counter; +} + +export function createGaugeMetric( + config: GaugeConfiguration, +): Gauge { + let metric = register.getSingleMetric(config.name); + if (!metric) { + metric = new Gauge(config); + register.registerMetric(metric); + } + return metric as Gauge; +} + +export function createSummaryMetric( + config: SummaryConfiguration, +): Summary { + let metric = register.getSingleMetric(config.name); + if (!metric) { + metric = new Summary(config); + register.registerMetric(metric); + } + + return metric as Summary; +} + +export function createHistogramMetric( + config: HistogramConfiguration, +): Histogram { + let metric = register.getSingleMetric(config.name); + if (!metric) { + metric = new Histogram(config); + register.registerMetric(metric); + } + + return metric as Histogram; +} diff --git a/yarn.lock b/yarn.lock index 359685cdd0..a661d39384 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6631,6 +6631,7 @@ __metadata: octokit: ^2.0.0 octokit-plugin-create-pull-request: ^3.10.0 p-limit: ^3.1.0 + prom-client: ^14.0.1 supertest: ^6.1.3 uuid: ^8.2.0 vm2: ^3.9.11