diff --git a/.changeset/silver-games-yell.md b/.changeset/silver-games-yell.md new file mode 100644 index 0000000000..3f9d3ec8bf --- /dev/null +++ b/.changeset/silver-games-yell.md @@ -0,0 +1,51 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +'@backstage/plugin-catalog-backend': patch +--- + +Added a set of default Prometheus metrics around scaffolding. See below for a list of metrics and an explanation of their labels: + +- `scaffolder_task_success_count`: Tracks successful task runs. + + Labels: + + - `template`: The entity ref of the scaffolded template + - `user`: The entity ref of the user that invoked the template run + +- scaffolder_task_error_count: a count that track how many task runs error out + + Labels: + + - `template`: The entity ref of the scaffolded template + - `user`: The entity ref of the user that invoked the template run + +- scaffolder_task_duration: a histogram which tracks the duration of a task run + + Labels: + + - `template`: The entity ref of the scaffolded template + - `result`: A boolean describing whether the task ran successfully + +- scaffolder_step_success_count: a count that tracks each step run + + Labels: + + - `template`: The entity ref of the scaffolded template + - `step`: The name of the step that was run + +- scaffolder_step_error_count: a count that tracks how many steps error out + + Labels: + + - `template`: The entity ref of the scaffolded template + - `step`: The name of the step that was run + +- scaffolder_step_duration: a histogram which tracks the duration of each step run + + Labels: + + - `template`: The entity ref of the scaffolded template + - `step`: The name of the step that was run + - `result`: A boolean describing whether the task ran successfully + +You can find a guide for running Prometheus metrics here: https://github.com/backstage/backstage/blob/master/contrib/docs/tutorials/prometheus-metrics.md diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 35f7fe01e2..b96be75294 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -72,6 +72,7 @@ "octokit": "^2.0.0", "octokit-plugin-create-pull-request": "^3.10.0", "p-limit": "^3.1.0", + "prom-client": "14.0.1", "uuid": "^8.2.0", "vm2": "^3.9.11", "winston": "^3.2.1", diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index 009ea1357d..f4776bc033 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -26,7 +26,7 @@ import { PassThrough } from 'stream'; import { generateExampleOutput, isTruthy } from './helper'; import { validate as validateJsonSchema } from 'jsonschema'; import { parseRepoUrl } from '../actions/builtin/publish/util'; -import { TemplateActionRegistry } from '../actions'; +import { TemplateAction, TemplateActionRegistry } from '../actions'; import { TemplateFilter, SecureTemplater, @@ -38,6 +38,7 @@ import { TaskStep, } from '@backstage/plugin-scaffolder-common'; import { UserEntity } from '@backstage/catalog-model'; +import { createCounterMetric, createHistogramMetric } from '../../util/metrics'; type NunjucksWorkflowRunnerOptions = { workingDirectory: string; @@ -96,6 +97,7 @@ const createStepLogger = ({ export class NunjucksWorkflowRunner implements WorkflowRunner { constructor(private readonly options: NunjucksWorkflowRunnerOptions) {} + private readonly tracker = scaffoldingTracker(); private isSingleTemplateString(input: string) { const { parser, nodes } = nunjucks as unknown as { @@ -200,10 +202,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { }); try { + const taskTrack = await this.tracker.taskStart(task); await fs.ensureDir(workspacePath); - await task.emitLog( - `Starting up task with ${task.spec.steps.length} steps`, - ); const context: TemplateContext = { parameters: task.spec.parameters, @@ -212,6 +212,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { }; for (const step of task.spec.steps) { + const stepTrack = await this.tracker.stepStart(task, step); try { if (step.if) { const ifResult = await this.render( @@ -220,23 +221,16 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { renderTemplate, ); if (!isTruthy(ifResult)) { - await task.emitLog( - `Skipping step ${step.id} because it's if condition was false`, - { stepId: step.id, status: 'skipped' }, - ); + await stepTrack.skipFalsy(); continue; } } - await task.emitLog(`Beginning step ${step.name}`, { - stepId: step.id, - status: 'processing', - }); - const action = this.options.actionRegistry.get(step.action); const { taskLogger, streamLogger } = createStepLogger({ task, step }); if (task.isDryRun) { + await taskTrack.skipDryRun(step, action); const redactedSecrets = Object.fromEntries( Object.entries(task.secrets ?? {}).map(secret => [ secret[0], @@ -339,20 +333,16 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { context.steps[step.id] = { output: stepOutput }; - await task.emitLog(`Finished step ${step.name}`, { - stepId: step.id, - status: 'completed', - }); + await stepTrack.markSuccessful(); } catch (err) { - await task.emitLog(String(err.stack), { - stepId: step.id, - status: 'failed', - }); + await taskTrack.markFailed(step, err); + stepTrack.markFailed(); throw err; } } const output = this.render(task.spec.output, context, renderTemplate); + taskTrack.markSuccessful(); return { output }; } finally { @@ -362,3 +352,134 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { } } } + +function scaffoldingTracker() { + const taskSuccesses = createCounterMetric({ + name: 'scaffolder_task_success_count', + help: 'Count of succesful task runs', + labelNames: ['template', 'user'], + }); + const taskErrors = createCounterMetric({ + name: 'scaffolder_task_error_count', + help: 'Count of failed task runs', + labelNames: ['template', 'user'], + }); + const taskDuration = createHistogramMetric({ + name: 'scaffolder_task_duration', + help: 'Duration of a task run', + labelNames: ['template', 'result'], + }); + const stepSuccesses = createCounterMetric({ + name: 'scaffolder_step_success_count', + help: 'Count of successful step runs', + labelNames: ['template', 'step'], + }); + const stepErrors = createCounterMetric({ + name: 'scaffolder_step_error_count', + help: 'Count of failed step runs', + labelNames: ['template', 'step'], + }); + const stepDuration = createHistogramMetric({ + name: 'scaffolder_step_duration', + help: 'Duration of a step runs', + labelNames: ['template', 'step', 'result'], + }); + + async function taskStart(task: TaskContext) { + await task.emitLog(`Starting up task with ${task.spec.steps.length} steps`); + const template = task.spec.templateInfo?.entityRef || ''; + const user = task.spec.user?.ref || ''; + + const taskTimer = taskDuration.startTimer({ + template, + }); + + async function skipDryRun( + step: TaskStep, + action: TemplateAction, + ) { + task.emitLog(`Skipping because ${action.id} does not support dry-run`, { + stepId: step.id, + status: 'skipped', + }); + } + + function markSuccessful() { + taskSuccesses.inc({ + template, + user, + }); + taskTimer({ result: 'ok' }); + } + + async function markFailed(step: TaskStep, err: Error) { + await task.emitLog(String(err.stack), { + stepId: step.id, + status: 'failed', + }); + taskErrors.inc({ + template, + user, + }); + taskTimer({ result: 'failed' }); + } + + return { + skipDryRun, + markSuccessful, + markFailed, + }; + } + + async function stepStart(task: TaskContext, step: TaskStep) { + await task.emitLog(`Beginning step ${step.name}`, { + stepId: step.id, + status: 'processing', + }); + const template = task.spec.templateInfo?.entityRef || ''; + + const stepTimer = stepDuration.startTimer({ + template, + step: step.name, + }); + + async function markSuccessful() { + await task.emitLog(`Finished step ${step.name}`, { + stepId: step.id, + status: 'completed', + }); + stepSuccesses.inc({ + template, + step: step.name, + }); + stepTimer({ result: 'ok' }); + } + + function markFailed() { + stepErrors.inc({ + template, + step: step.name, + }); + stepTimer({ result: 'failed' }); + } + + async function skipFalsy() { + await task.emitLog( + `Skipping step ${step.id} because it's if condition was false`, + { stepId: step.id, status: 'skipped' }, + ); + stepTimer({ result: 'skipped' }); + } + + return { + markSuccessful, + markFailed, + skipFalsy, + }; + } + + return { + taskStart, + stepStart, + }; +} diff --git a/plugins/scaffolder-backend/src/scaffolder/util/metrics.ts b/plugins/scaffolder-backend/src/scaffolder/util/metrics.ts new file mode 100644 index 0000000000..207135f234 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/util/metrics.ts @@ -0,0 +1,73 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Counter, + CounterConfiguration, + Gauge, + GaugeConfiguration, + Histogram, + HistogramConfiguration, + register, + Summary, + SummaryConfiguration, +} from 'prom-client'; + +export function createCounterMetric( + config: CounterConfiguration, +): Counter { + let metric = register.getSingleMetric(config.name); + if (!metric) { + metric = new Counter(config); + register.registerMetric(metric); + } + return metric as Counter; +} + +export function createGaugeMetric( + config: GaugeConfiguration, +): Gauge { + let metric = register.getSingleMetric(config.name); + if (!metric) { + metric = new Gauge(config); + register.registerMetric(metric); + } + return metric as Gauge; +} + +export function createSummaryMetric( + config: SummaryConfiguration, +): Summary { + let metric = register.getSingleMetric(config.name); + if (!metric) { + metric = new Summary(config); + register.registerMetric(metric); + } + + return metric as Summary; +} + +export function createHistogramMetric( + config: HistogramConfiguration, +): Histogram { + let metric = register.getSingleMetric(config.name); + if (!metric) { + metric = new Histogram(config); + register.registerMetric(metric); + } + + return metric as Histogram; +} diff --git a/yarn.lock b/yarn.lock index 1632a3a59e..774ad8e3ce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6395,6 +6395,7 @@ __metadata: octokit: ^2.0.0 octokit-plugin-create-pull-request: ^3.10.0 p-limit: ^3.1.0 + prom-client: 14.0.1 supertest: ^6.1.3 uuid: ^8.2.0 vm2: ^3.9.11 @@ -32362,6 +32363,15 @@ __metadata: languageName: node linkType: hard +"prom-client@npm:14.0.1": + version: 14.0.1 + resolution: "prom-client@npm:14.0.1" + dependencies: + tdigest: ^0.1.1 + checksum: 864c19b7086eda8fae652385bc8b8aeb155f85922e58672d07a64918a603341e120e65e08f9d77ccab546518dc18930284da8743c2aac3c968f626d7063d6bba + languageName: node + linkType: hard + "prom-client@npm:^14.0.1": version: 14.1.0 resolution: "prom-client@npm:14.1.0"