Merge pull request #13815 from spencerrichardhenry/scaffolderPromMetrics

Add default prom metrics to scaffolder
2022-10-21 20:40:06 +02:00
parent 983df167dd fc1edff210
commit f0de00d4eb
6 changed files with 253 additions and 28 deletions
@@ -0,0 +1,38 @@
+---
+'@backstage/plugin-scaffolder-backend': minor
+---
+
+Added a set of default Prometheus metrics around scaffolding. See below for a list of metrics and an explanation of their labels:
+
+- `scaffolder_task_count`: Tracks successful task runs.
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `user`: The entity ref of the user that invoked the template run
+  - `result`: A string describing whether the task ran successfully, failed, or was skipped
+
+- `scaffolder_task_duration`: a histogram which tracks the duration of a task run
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `result`: A boolean describing whether the task ran successfully
+
+- `scaffolder_step_count`: a count that tracks each step run
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `step`: The name of the step that was run
+  - `result`: A string describing whether the task ran successfully, failed, or was skipped
+
+- `scaffolder_step_duration`: a histogram which tracks the duration of each step run
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `step`: The name of the step that was run
+  - `result`: A string describing whether the task ran successfully, failed, or was skipped
+
+You can find a guide for running Prometheus metrics here: https://github.com/backstage/backstage/blob/master/contrib/docs/tutorials/prometheus-metrics.md
@@ -106,3 +106,7 @@ There are some custom metrics that have been added to Backstage will be output f
 - `catalog_processing_duration_seconds`: Time spent executing the full processing flow
 - `catalog_processors_duration_seconds`: Time spent executing catalog processors
 - `catalog_processing_queue_delay_seconds`: The amount of delay between being scheduled for processing, and the start of actually being processed
+- `scaffolder_task_count`: Tracks successful task runs.
+- `scaffolder_task_duration`: a histogram which tracks the duration of a task run
+- `scaffolder_step_count`: a count that tracks each step run
+- `scaffolder_step_duration`: a histogram which tracks the duration of each step run
@@ -72,6 +72,7 @@
    "octokit": "^2.0.0",
    "octokit-plugin-create-pull-request": "^3.10.0",
    "p-limit": "^3.1.0",
+    "prom-client": "^14.0.1",
    "uuid": "^8.2.0",
    "vm2": "^3.9.11",
    "winston": "^3.2.1",
@@ -26,7 +26,7 @@ import { PassThrough } from 'stream';
 import { generateExampleOutput, isTruthy } from './helper';
 import { validate as validateJsonSchema } from 'jsonschema';
 import { parseRepoUrl } from '../actions/builtin/publish/util';
-import { TemplateActionRegistry } from '../actions';
+import { TemplateAction, TemplateActionRegistry } from '../actions';
 import {
  TemplateFilter,
  SecureTemplater,
@@ -39,6 +39,7 @@ import {
  TaskStep,
 } from '@backstage/plugin-scaffolder-common';
 import { UserEntity } from '@backstage/catalog-model';
+import { createCounterMetric, createHistogramMetric } from '../../util/metrics';

 type NunjucksWorkflowRunnerOptions = {
  workingDirectory: string;
@@ -97,6 +98,7 @@ const createStepLogger = ({

 export class NunjucksWorkflowRunner implements WorkflowRunner {
  constructor(private readonly options: NunjucksWorkflowRunnerOptions) {}
+  private readonly tracker = scaffoldingTracker();

  private isSingleTemplateString(input: string) {
    const { parser, nodes } = nunjucks as unknown as {
@@ -202,10 +204,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
    });

    try {
+      const taskTrack = await this.tracker.taskStart(task);
      await fs.ensureDir(workspacePath);
-      await task.emitLog(
-        `Starting up task with ${task.spec.steps.length} steps`,
-      );

      const context: TemplateContext = {
        parameters: task.spec.parameters,
@@ -214,6 +214,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
      };

      for (const step of task.spec.steps) {
+        const stepTrack = await this.tracker.stepStart(task, step);
        try {
          if (step.if) {
            const ifResult = await this.render(
@@ -222,19 +223,11 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
              renderTemplate,
            );
            if (!isTruthy(ifResult)) {
-              await task.emitLog(
-                `Skipping step ${step.id} because it's if condition was false`,
-                { stepId: step.id, status: 'skipped' },
-              );
+              await stepTrack.skipFalsy();
              continue;
            }
          }

-          await task.emitLog(`Beginning step ${step.name}`, {
-            stepId: step.id,
-            status: 'processing',
-          });
-
          const action = this.options.actionRegistry.get(step.action);
          const { taskLogger, streamLogger } = createStepLogger({ task, step });

@@ -266,13 +259,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
              )}`,
            );
            if (!action.supportsDryRun) {
-              task.emitLog(
-                `Skipping because ${action.id} does not support dry-run`,
-                {
-                  stepId: step.id,
-                  status: 'skipped',
-                },
-              );
+              await taskTrack.skipDryRun(step, action);
              const outputSchema = action.schema?.output;
              if (outputSchema) {
                context.steps[step.id] = {
@@ -341,20 +328,16 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {

          context.steps[step.id] = { output: stepOutput };

-          await task.emitLog(`Finished step ${step.name}`, {
-            stepId: step.id,
-            status: 'completed',
-          });
+          await stepTrack.markSuccessful();
        } catch (err) {
-          await task.emitLog(String(err.stack), {
-            stepId: step.id,
-            status: 'failed',
-          });
+          await taskTrack.markFailed(step, err);
+          await stepTrack.markFailed();
          throw err;
        }
      }

      const output = this.render(task.spec.output, context, renderTemplate);
+      await taskTrack.markSuccessful();

      return { output };
    } finally {
@@ -364,3 +347,128 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
    }
  }
 }
+
+function scaffoldingTracker() {
+  const taskCount = createCounterMetric({
+    name: 'scaffolder_task_count',
+    help: 'Count of task runs',
+    labelNames: ['template', 'user', 'result'],
+  });
+  const taskDuration = createHistogramMetric({
+    name: 'scaffolder_task_duration',
+    help: 'Duration of a task run',
+    labelNames: ['template', 'result'],
+  });
+  const stepCount = createCounterMetric({
+    name: 'scaffolder_step_count',
+    help: 'Count of step runs',
+    labelNames: ['template', 'step', 'result'],
+  });
+  const stepDuration = createHistogramMetric({
+    name: 'scaffolder_step_duration',
+    help: 'Duration of a step runs',
+    labelNames: ['template', 'step', 'result'],
+  });
+
+  async function taskStart(task: TaskContext) {
+    await task.emitLog(`Starting up task with ${task.spec.steps.length} steps`);
+    const template = task.spec.templateInfo?.entityRef || '';
+    const user = task.spec.user?.ref || '';
+
+    const taskTimer = taskDuration.startTimer({
+      template,
+    });
+
+    async function skipDryRun(
+      step: TaskStep,
+      action: TemplateAction<JsonObject>,
+    ) {
+      task.emitLog(`Skipping because ${action.id} does not support dry-run`, {
+        stepId: step.id,
+        status: 'skipped',
+      });
+    }
+
+    async function markSuccessful() {
+      taskCount.inc({
+        template,
+        user,
+        result: 'ok',
+      });
+      taskTimer({ result: 'ok' });
+    }
+
+    async function markFailed(step: TaskStep, err: Error) {
+      await task.emitLog(String(err.stack), {
+        stepId: step.id,
+        status: 'failed',
+      });
+      taskCount.inc({
+        template,
+        user,
+        result: 'failed',
+      });
+      taskTimer({ result: 'failed' });
+    }
+
+    return {
+      skipDryRun,
+      markSuccessful,
+      markFailed,
+    };
+  }
+
+  async function stepStart(task: TaskContext, step: TaskStep) {
+    await task.emitLog(`Beginning step ${step.name}`, {
+      stepId: step.id,
+      status: 'processing',
+    });
+    const template = task.spec.templateInfo?.entityRef || '';
+
+    const stepTimer = stepDuration.startTimer({
+      template,
+      step: step.name,
+    });
+
+    async function markSuccessful() {
+      await task.emitLog(`Finished step ${step.name}`, {
+        stepId: step.id,
+        status: 'completed',
+      });
+      stepCount.inc({
+        template,
+        step: step.name,
+        result: 'ok',
+      });
+      stepTimer({ result: 'ok' });
+    }
+
+    async function markFailed() {
+      stepCount.inc({
+        template,
+        step: step.name,
+        result: 'failed',
+      });
+      stepTimer({ result: 'failed' });
+    }
+
+    async function skipFalsy() {
+      await task.emitLog(
+        `Skipping step ${step.id} because its if condition was false`,
+        { stepId: step.id, status: 'skipped' },
+      );
+      stepTimer({ result: 'skipped' });
+    }
+
+    return {
+      markSuccessful,
+      markFailed,
+      skipFalsy,
+    };
+  }
+
+  return {
+    taskStart,
+    stepStart,
+  };
+}
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  Counter,
+  CounterConfiguration,
+  Gauge,
+  GaugeConfiguration,
+  Histogram,
+  HistogramConfiguration,
+  register,
+  Summary,
+  SummaryConfiguration,
+} from 'prom-client';
+
+export function createCounterMetric<T extends string>(
+  config: CounterConfiguration<T>,
+): Counter<T> {
+  let metric = register.getSingleMetric(config.name);
+  if (!metric) {
+    metric = new Counter<T>(config);
+    register.registerMetric(metric);
+  }
+  return metric as Counter<T>;
+}
+
+export function createGaugeMetric<T extends string>(
+  config: GaugeConfiguration<T>,
+): Gauge<T> {
+  let metric = register.getSingleMetric(config.name);
+  if (!metric) {
+    metric = new Gauge<T>(config);
+    register.registerMetric(metric);
+  }
+  return metric as Gauge<T>;
+}
+
+export function createSummaryMetric<T extends string>(
+  config: SummaryConfiguration<T>,
+): Summary<T> {
+  let metric = register.getSingleMetric(config.name);
+  if (!metric) {
+    metric = new Summary<T>(config);
+    register.registerMetric(metric);
+  }
+
+  return metric as Summary<T>;
+}
+
+export function createHistogramMetric<T extends string>(
+  config: HistogramConfiguration<T>,
+): Histogram<T> {
+  let metric = register.getSingleMetric(config.name);
+  if (!metric) {
+    metric = new Histogram<T>(config);
+    register.registerMetric(metric);
+  }
+
+  return metric as Histogram<T>;
+}
@@ -6631,6 +6631,7 @@ __metadata:
    octokit: ^2.0.0
    octokit-plugin-create-pull-request: ^3.10.0
    p-limit: ^3.1.0
+    prom-client: ^14.0.1
    supertest: ^6.1.3
    uuid: ^8.2.0
    vm2: ^3.9.11