Add default prom metrics to scaffolder

Signed-off-by: Spencer Henry <shenry@qualtrics.com>

.changeset/silver-games-yell.md

@@ -0,0 +1,51 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+'@backstage/plugin-catalog-backend': patch
+---
+
+Added a set of default Prometheus metrics around scaffolding. See below for a list of metrics and an explanation of their labels:
+
+- `scaffolder_task_success_count`: Tracks successful task runs.
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `user`: The entity ref of the user that invoked the template run
+
+- scaffolder_task_error_count: a count that track how many task runs error out
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `user`: The entity ref of the user that invoked the template run
+
+- scaffolder_task_duration: a histogram which tracks the duration of a task run
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `result`: A boolean describing whether the task ran successfully
+
+- scaffolder_step_success_count: a count that tracks each step run
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `step`: The name of the step that was run
+
+- scaffolder_step_error_count: a count that tracks how many steps error out
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `step`: The name of the step that was run
+
+- scaffolder_step_duration: a histogram which tracks the duration of each step run
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `step`: The name of the step that was run
+  - `result`: A boolean describing whether the task ran successfully
+
+You can find a guide for running Prometheus metrics here: https://github.com/backstage/backstage/blob/master/contrib/docs/tutorials/prometheus-metrics.md

plugins/scaffolder-backend/package.json

@@ -72,6 +72,7 @@
     "octokit": "^2.0.0",
     "octokit-plugin-create-pull-request": "^3.10.0",
     "p-limit": "^3.1.0",
+    "prom-client": "14.0.1",
     "uuid": "^8.2.0",
     "vm2": "^3.9.11",
     "winston": "^3.2.1",

plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts

@@ -26,7 +26,7 @@ import { PassThrough } from 'stream';
 import { generateExampleOutput, isTruthy } from './helper';
 import { validate as validateJsonSchema } from 'jsonschema';
 import { parseRepoUrl } from '../actions/builtin/publish/util';
-import { TemplateActionRegistry } from '../actions';
+import { TemplateAction, TemplateActionRegistry } from '../actions';
 import {
   TemplateFilter,
   SecureTemplater,
@@ -38,6 +38,7 @@ import {
   TaskStep,
 } from '@backstage/plugin-scaffolder-common';
 import { UserEntity } from '@backstage/catalog-model';
+import { createCounterMetric, createHistogramMetric } from '../../util/metrics';
 
 type NunjucksWorkflowRunnerOptions = {
   workingDirectory: string;
@@ -96,6 +97,7 @@ const createStepLogger = ({
 
 export class NunjucksWorkflowRunner implements WorkflowRunner {
   constructor(private readonly options: NunjucksWorkflowRunnerOptions) {}
+  private readonly tracker = scaffoldingTracker();
 
   private isSingleTemplateString(input: string) {
     const { parser, nodes } = nunjucks as unknown as {
@@ -200,10 +202,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
     });
 
     try {
+      const taskTrack = await this.tracker.taskStart(task);
       await fs.ensureDir(workspacePath);
-      await task.emitLog(
-        `Starting up task with ${task.spec.steps.length} steps`,
-      );
 
       const context: TemplateContext = {
         parameters: task.spec.parameters,
@@ -212,6 +212,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
       };
 
       for (const step of task.spec.steps) {
+        const stepTrack = await this.tracker.stepStart(task, step);
         try {
           if (step.if) {
             const ifResult = await this.render(
@@ -220,23 +221,16 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
               renderTemplate,
             );
             if (!isTruthy(ifResult)) {
-              await task.emitLog(
-                `Skipping step ${step.id} because it's if condition was false`,
-                { stepId: step.id, status: 'skipped' },
-              );
+              await stepTrack.skipFalsy();
               continue;
             }
           }
 
-          await task.emitLog(`Beginning step ${step.name}`, {
-            stepId: step.id,
-            status: 'processing',
-          });
-
           const action = this.options.actionRegistry.get(step.action);
           const { taskLogger, streamLogger } = createStepLogger({ task, step });
 
           if (task.isDryRun) {
+            await taskTrack.skipDryRun(step, action);
             const redactedSecrets = Object.fromEntries(
               Object.entries(task.secrets ?? {}).map(secret => [
                 secret[0],
@@ -339,20 +333,16 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
 
           context.steps[step.id] = { output: stepOutput };
 
-          await task.emitLog(`Finished step ${step.name}`, {
-            stepId: step.id,
-            status: 'completed',
-          });
+          await stepTrack.markSuccessful();
         } catch (err) {
-          await task.emitLog(String(err.stack), {
-            stepId: step.id,
-            status: 'failed',
-          });
+          await taskTrack.markFailed(step, err);
+          stepTrack.markFailed();
           throw err;
         }
       }
 
       const output = this.render(task.spec.output, context, renderTemplate);
+      taskTrack.markSuccessful();
 
       return { output };
     } finally {
@@ -362,3 +352,134 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
     }
   }
 }
+
+function scaffoldingTracker() {
+  const taskSuccesses = createCounterMetric({
+    name: 'scaffolder_task_success_count',
+    help: 'Count of succesful task runs',
+    labelNames: ['template', 'user'],
+  });
+  const taskErrors = createCounterMetric({
+    name: 'scaffolder_task_error_count',
+    help: 'Count of failed task runs',
+    labelNames: ['template', 'user'],
+  });
+  const taskDuration = createHistogramMetric({
+    name: 'scaffolder_task_duration',
+    help: 'Duration of a task run',
+    labelNames: ['template', 'result'],
+  });
+  const stepSuccesses = createCounterMetric({
+    name: 'scaffolder_step_success_count',
+    help: 'Count of successful step runs',
+    labelNames: ['template', 'step'],
+  });
+  const stepErrors = createCounterMetric({
+    name: 'scaffolder_step_error_count',
+    help: 'Count of failed step runs',
+    labelNames: ['template', 'step'],
+  });
+  const stepDuration = createHistogramMetric({
+    name: 'scaffolder_step_duration',
+    help: 'Duration of a step runs',
+    labelNames: ['template', 'step', 'result'],
+  });
+
+  async function taskStart(task: TaskContext) {
+    await task.emitLog(`Starting up task with ${task.spec.steps.length} steps`);
+    const template = task.spec.templateInfo?.entityRef || '';
+    const user = task.spec.user?.ref || '';
+
+    const taskTimer = taskDuration.startTimer({
+      template,
+    });
+
+    async function skipDryRun(
+      step: TaskStep,
+      action: TemplateAction<JsonObject>,
+    ) {
+      task.emitLog(`Skipping because ${action.id} does not support dry-run`, {
+        stepId: step.id,
+        status: 'skipped',
+      });
+    }
+
+    function markSuccessful() {
+      taskSuccesses.inc({
+        template,
+        user,
+      });
+      taskTimer({ result: 'ok' });
+    }
+
+    async function markFailed(step: TaskStep, err: Error) {
+      await task.emitLog(String(err.stack), {
+        stepId: step.id,
+        status: 'failed',
+      });
+      taskErrors.inc({
+        template,
+        user,
+      });
+      taskTimer({ result: 'failed' });
+    }
+
+    return {
+      skipDryRun,
+      markSuccessful,
+      markFailed,
+    };
+  }
+
+  async function stepStart(task: TaskContext, step: TaskStep) {
+    await task.emitLog(`Beginning step ${step.name}`, {
+      stepId: step.id,
+      status: 'processing',
+    });
+    const template = task.spec.templateInfo?.entityRef || '';
+
+    const stepTimer = stepDuration.startTimer({
+      template,
+      step: step.name,
+    });
+
+    async function markSuccessful() {
+      await task.emitLog(`Finished step ${step.name}`, {
+        stepId: step.id,
+        status: 'completed',
+      });
+      stepSuccesses.inc({
+        template,
+        step: step.name,
+      });
+      stepTimer({ result: 'ok' });
+    }
+
+    function markFailed() {
+      stepErrors.inc({
+        template,
+        step: step.name,
+      });
+      stepTimer({ result: 'failed' });
+    }
+
+    async function skipFalsy() {
+      await task.emitLog(
+        `Skipping step ${step.id} because it's if condition was false`,
+        { stepId: step.id, status: 'skipped' },
+      );
+      stepTimer({ result: 'skipped' });
+    }
+
+    return {
+      markSuccessful,
+      markFailed,
+      skipFalsy,
+    };
+  }
+
+  return {
+    taskStart,
+    stepStart,
+  };
+}

plugins/scaffolder-backend/src/scaffolder/util/metrics.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  Counter,
+  CounterConfiguration,
+  Gauge,
+  GaugeConfiguration,
+  Histogram,
+  HistogramConfiguration,
+  register,
+  Summary,
+  SummaryConfiguration,
+} from 'prom-client';
+
+export function createCounterMetric<T extends string>(
+  config: CounterConfiguration<T>,
+): Counter<T> {
+  let metric = register.getSingleMetric(config.name);
+  if (!metric) {
+    metric = new Counter<T>(config);
+    register.registerMetric(metric);
+  }
+  return metric as Counter<T>;
+}
+
+export function createGaugeMetric<T extends string>(
+  config: GaugeConfiguration<T>,
+): Gauge<T> {
+  let metric = register.getSingleMetric(config.name);
+  if (!metric) {
+    metric = new Gauge<T>(config);
+    register.registerMetric(metric);
+  }
+  return metric as Gauge<T>;
+}
+
+export function createSummaryMetric<T extends string>(
+  config: SummaryConfiguration<T>,
+): Summary<T> {
+  let metric = register.getSingleMetric(config.name);
+  if (!metric) {
+    metric = new Summary<T>(config);
+    register.registerMetric(metric);
+  }
+
+  return metric as Summary<T>;
+}
+
+export function createHistogramMetric<T extends string>(
+  config: HistogramConfiguration<T>,
+): Histogram<T> {
+  let metric = register.getSingleMetric(config.name);
+  if (!metric) {
+    metric = new Histogram<T>(config);
+    register.registerMetric(metric);
+  }
+
+  return metric as Histogram<T>;
+}

yarn.lock

@@ -6395,6 +6395,7 @@ __metadata:
     octokit: ^2.0.0
     octokit-plugin-create-pull-request: ^3.10.0
     p-limit: ^3.1.0
+    prom-client: 14.0.1
     supertest: ^6.1.3
     uuid: ^8.2.0
     vm2: ^3.9.11
@@ -32362,6 +32363,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"prom-client@npm:14.0.1":
+  version: 14.0.1
+  resolution: "prom-client@npm:14.0.1"
+  dependencies:
+    tdigest: ^0.1.1
+  checksum: 864c19b7086eda8fae652385bc8b8aeb155f85922e58672d07a64918a603341e120e65e08f9d77ccab546518dc18930284da8743c2aac3c968f626d7063d6bba
+  languageName: node
+  linkType: hard
+
 "prom-client@npm:^14.0.1":
   version: 14.1.0
   resolution: "prom-client@npm:14.1.0"