From f74aa0c08d449defe77501800389c84e3065d7ed Mon Sep 17 00:00:00 2001
From: Patrik Oldsberg <poldsberg@gmail.com>
Date: Thu, 22 Sep 2022 15:18:59 +0200
Subject: [PATCH 01/18] backend-tests: increase timeout for setting up test
 databases

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>
Signed-off-by: Spencer Henry <shenry@qualtrics.com>
---
 .../backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts
index f6177b57f7..d61948a929 100644
--- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts
+++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts
@@ -45,7 +45,7 @@ describe('PluginTaskManagerImpl', () => {
     );
 
     jest.useFakeTimers();
-  }, 30_000);
+  }, 60_000);
 
   async function init(databaseId: TestDatabaseId) {
     const knex = await databases.init(databaseId);

From a6075e94b5a39e96b604d2be79b30ca4dd857b71 Mon Sep 17 00:00:00 2001
From: Spencer Henry <shenry@qualtrics.com>
Date: Thu, 22 Sep 2022 12:58:41 -0600
Subject: [PATCH 02/18] Add default prom metrics to scaffolder

Signed-off-by: Spencer Henry <shenry@qualtrics.com>
---
 .changeset/silver-games-yell.md               |  51 ++++++
 plugins/scaffolder-backend/package.json       |   1 +
 .../tasks/NunjucksWorkflowRunner.ts           | 163 +++++++++++++++---
 .../src/scaffolder/util/metrics.ts            |  73 ++++++++
 yarn.lock                                     |  10 ++
 5 files changed, 277 insertions(+), 21 deletions(-)
 create mode 100644 .changeset/silver-games-yell.md
 create mode 100644 plugins/scaffolder-backend/src/scaffolder/util/metrics.ts

diff --git a/.changeset/silver-games-yell.md b/.changeset/silver-games-yell.md
new file mode 100644
index 0000000000..3f9d3ec8bf
--- /dev/null
+++ b/.changeset/silver-games-yell.md
@@ -0,0 +1,51 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+'@backstage/plugin-catalog-backend': patch
+---
+
+Added a set of default Prometheus metrics around scaffolding. See below for a list of metrics and an explanation of their labels:
+
+- `scaffolder_task_success_count`: Tracks successful task runs.
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `user`: The entity ref of the user that invoked the template run
+
+- scaffolder_task_error_count: a count that track how many task runs error out
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `user`: The entity ref of the user that invoked the template run
+
+- scaffolder_task_duration: a histogram which tracks the duration of a task run
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `result`: A boolean describing whether the task ran successfully
+
+- scaffolder_step_success_count: a count that tracks each step run
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `step`: The name of the step that was run
+
+- scaffolder_step_error_count: a count that tracks how many steps error out
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `step`: The name of the step that was run
+
+- scaffolder_step_duration: a histogram which tracks the duration of each step run
+
+  Labels:
+
+  - `template`: The entity ref of the scaffolded template
+  - `step`: The name of the step that was run
+  - `result`: A boolean describing whether the task ran successfully
+
+You can find a guide for running Prometheus metrics here: https://github.com/backstage/backstage/blob/master/contrib/docs/tutorials/prometheus-metrics.md
diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json
index 35f7fe01e2..b96be75294 100644
--- a/plugins/scaffolder-backend/package.json
+++ b/plugins/scaffolder-backend/package.json
@@ -72,6 +72,7 @@
     "octokit": "^2.0.0",
     "octokit-plugin-create-pull-request": "^3.10.0",
     "p-limit": "^3.1.0",
+    "prom-client": "14.0.1",
     "uuid": "^8.2.0",
     "vm2": "^3.9.11",
     "winston": "^3.2.1",
diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
index 009ea1357d..f4776bc033 100644
--- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
+++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
@@ -26,7 +26,7 @@ import { PassThrough } from 'stream';
 import { generateExampleOutput, isTruthy } from './helper';
 import { validate as validateJsonSchema } from 'jsonschema';
 import { parseRepoUrl } from '../actions/builtin/publish/util';
-import { TemplateActionRegistry } from '../actions';
+import { TemplateAction, TemplateActionRegistry } from '../actions';
 import {
   TemplateFilter,
   SecureTemplater,
@@ -38,6 +38,7 @@ import {
   TaskStep,
 } from '@backstage/plugin-scaffolder-common';
 import { UserEntity } from '@backstage/catalog-model';
+import { createCounterMetric, createHistogramMetric } from '../../util/metrics';
 
 type NunjucksWorkflowRunnerOptions = {
   workingDirectory: string;
@@ -96,6 +97,7 @@ const createStepLogger = ({
 
 export class NunjucksWorkflowRunner implements WorkflowRunner {
   constructor(private readonly options: NunjucksWorkflowRunnerOptions) {}
+  private readonly tracker = scaffoldingTracker();
 
   private isSingleTemplateString(input: string) {
     const { parser, nodes } = nunjucks as unknown as {
@@ -200,10 +202,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
     });
 
     try {
+      const taskTrack = await this.tracker.taskStart(task);
       await fs.ensureDir(workspacePath);
-      await task.emitLog(
-        `Starting up task with ${task.spec.steps.length} steps`,
-      );
 
       const context: TemplateContext = {
         parameters: task.spec.parameters,
@@ -212,6 +212,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
       };
 
       for (const step of task.spec.steps) {
+        const stepTrack = await this.tracker.stepStart(task, step);
         try {
           if (step.if) {
             const ifResult = await this.render(
@@ -220,23 +221,16 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
               renderTemplate,
             );
             if (!isTruthy(ifResult)) {
-              await task.emitLog(
-                `Skipping step ${step.id} because it's if condition was false`,
-                { stepId: step.id, status: 'skipped' },
-              );
+              await stepTrack.skipFalsy();
               continue;
             }
           }
 
-          await task.emitLog(`Beginning step ${step.name}`, {
-            stepId: step.id,
-            status: 'processing',
-          });
-
           const action = this.options.actionRegistry.get(step.action);
           const { taskLogger, streamLogger } = createStepLogger({ task, step });
 
           if (task.isDryRun) {
+            await taskTrack.skipDryRun(step, action);
             const redactedSecrets = Object.fromEntries(
               Object.entries(task.secrets ?? {}).map(secret => [
                 secret[0],
@@ -339,20 +333,16 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
 
           context.steps[step.id] = { output: stepOutput };
 
-          await task.emitLog(`Finished step ${step.name}`, {
-            stepId: step.id,
-            status: 'completed',
-          });
+          await stepTrack.markSuccessful();
         } catch (err) {
-          await task.emitLog(String(err.stack), {
-            stepId: step.id,
-            status: 'failed',
-          });
+          await taskTrack.markFailed(step, err);
+          stepTrack.markFailed();
           throw err;
         }
       }
 
       const output = this.render(task.spec.output, context, renderTemplate);
+      taskTrack.markSuccessful();
 
       return { output };
     } finally {
@@ -362,3 +352,134 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
     }
   }
 }
+
+function scaffoldingTracker() {
+  const taskSuccesses = createCounterMetric({
+    name: 'scaffolder_task_success_count',
+    help: 'Count of succesful task runs',
+    labelNames: ['template', 'user'],
+  });
+  const taskErrors = createCounterMetric({
+    name: 'scaffolder_task_error_count',
+    help: 'Count of failed task runs',
+    labelNames: ['template', 'user'],
+  });
+  const taskDuration = createHistogramMetric({
+    name: 'scaffolder_task_duration',
+    help: 'Duration of a task run',
+    labelNames: ['template', 'result'],
+  });
+  const stepSuccesses = createCounterMetric({
+    name: 'scaffolder_step_success_count',
+    help: 'Count of successful step runs',
+    labelNames: ['template', 'step'],
+  });
+  const stepErrors = createCounterMetric({
+    name: 'scaffolder_step_error_count',
+    help: 'Count of failed step runs',
+    labelNames: ['template', 'step'],
+  });
+  const stepDuration = createHistogramMetric({
+    name: 'scaffolder_step_duration',
+    help: 'Duration of a step runs',
+    labelNames: ['template', 'step', 'result'],
+  });
+
+  async function taskStart(task: TaskContext) {
+    await task.emitLog(`Starting up task with ${task.spec.steps.length} steps`);
+    const template = task.spec.templateInfo?.entityRef || '';
+    const user = task.spec.user?.ref || '';
+
+    const taskTimer = taskDuration.startTimer({
+      template,
+    });
+
+    async function skipDryRun(
+      step: TaskStep,
+      action: TemplateAction<JsonObject>,
+    ) {
+      task.emitLog(`Skipping because ${action.id} does not support dry-run`, {
+        stepId: step.id,
+        status: 'skipped',
+      });
+    }
+
+    function markSuccessful() {
+      taskSuccesses.inc({
+        template,
+        user,
+      });
+      taskTimer({ result: 'ok' });
+    }
+
+    async function markFailed(step: TaskStep, err: Error) {
+      await task.emitLog(String(err.stack), {
+        stepId: step.id,
+        status: 'failed',
+      });
+      taskErrors.inc({
+        template,
+        user,
+      });
+      taskTimer({ result: 'failed' });
+    }
+
+    return {
+      skipDryRun,
+      markSuccessful,
+      markFailed,
+    };
+  }
+
+  async function stepStart(task: TaskContext, step: TaskStep) {
+    await task.emitLog(`Beginning step ${step.name}`, {
+      stepId: step.id,
+      status: 'processing',
+    });
+    const template = task.spec.templateInfo?.entityRef || '';
+
+    const stepTimer = stepDuration.startTimer({
+      template,
+      step: step.name,
+    });
+
+    async function markSuccessful() {
+      await task.emitLog(`Finished step ${step.name}`, {
+        stepId: step.id,
+        status: 'completed',
+      });
+      stepSuccesses.inc({
+        template,
+        step: step.name,
+      });
+      stepTimer({ result: 'ok' });
+    }
+
+    function markFailed() {
+      stepErrors.inc({
+        template,
+        step: step.name,
+      });
+      stepTimer({ result: 'failed' });
+    }
+
+    async function skipFalsy() {
+      await task.emitLog(
+        `Skipping step ${step.id} because it's if condition was false`,
+        { stepId: step.id, status: 'skipped' },
+      );
+      stepTimer({ result: 'skipped' });
+    }
+
+    return {
+      markSuccessful,
+      markFailed,
+      skipFalsy,
+    };
+  }
+
+  return {
+    taskStart,
+    stepStart,
+  };
+}
diff --git a/plugins/scaffolder-backend/src/scaffolder/util/metrics.ts b/plugins/scaffolder-backend/src/scaffolder/util/metrics.ts
new file mode 100644
index 0000000000..207135f234
--- /dev/null
+++ b/plugins/scaffolder-backend/src/scaffolder/util/metrics.ts
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  Counter,
+  CounterConfiguration,
+  Gauge,
+  GaugeConfiguration,
+  Histogram,
+  HistogramConfiguration,
+  register,
+  Summary,
+  SummaryConfiguration,
+} from 'prom-client';
+
+export function createCounterMetric<T extends string>(
+  config: CounterConfiguration<T>,
+): Counter<T> {
+  let metric = register.getSingleMetric(config.name);
+  if (!metric) {
+    metric = new Counter<T>(config);
+    register.registerMetric(metric);
+  }
+  return metric as Counter<T>;
+}
+
+export function createGaugeMetric<T extends string>(
+  config: GaugeConfiguration<T>,
+): Gauge<T> {
+  let metric = register.getSingleMetric(config.name);
+  if (!metric) {
+    metric = new Gauge<T>(config);
+    register.registerMetric(metric);
+  }
+  return metric as Gauge<T>;
+}
+
+export function createSummaryMetric<T extends string>(
+  config: SummaryConfiguration<T>,
+): Summary<T> {
+  let metric = register.getSingleMetric(config.name);
+  if (!metric) {
+    metric = new Summary<T>(config);
+    register.registerMetric(metric);
+  }
+
+  return metric as Summary<T>;
+}
+
+export function createHistogramMetric<T extends string>(
+  config: HistogramConfiguration<T>,
+): Histogram<T> {
+  let metric = register.getSingleMetric(config.name);
+  if (!metric) {
+    metric = new Histogram<T>(config);
+    register.registerMetric(metric);
+  }
+
+  return metric as Histogram<T>;
+}
diff --git a/yarn.lock b/yarn.lock
index 1632a3a59e..774ad8e3ce 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6395,6 +6395,7 @@ __metadata:
     octokit: ^2.0.0
     octokit-plugin-create-pull-request: ^3.10.0
     p-limit: ^3.1.0
+    prom-client: 14.0.1
     supertest: ^6.1.3
     uuid: ^8.2.0
     vm2: ^3.9.11
@@ -32362,6 +32363,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"prom-client@npm:14.0.1":
+  version: 14.0.1
+  resolution: "prom-client@npm:14.0.1"
+  dependencies:
+    tdigest: ^0.1.1
+  checksum: 864c19b7086eda8fae652385bc8b8aeb155f85922e58672d07a64918a603341e120e65e08f9d77ccab546518dc18930284da8743c2aac3c968f626d7063d6bba
+  languageName: node
+  linkType: hard
+
 "prom-client@npm:^14.0.1":
   version: 14.1.0
   resolution: "prom-client@npm:14.1.0"

From 685b693fc0e9ce5d59f5883e1c4a7e43bd51d111 Mon Sep 17 00:00:00 2001
From: Spencer Henry <shenry@qualtrics.com>
Date: Thu, 22 Sep 2022 13:32:53 -0600
Subject: [PATCH 03/18] move util folder up a dir

Signed-off-by: Spencer Henry <shenry@qualtrics.com>
---
 plugins/scaffolder-backend/src/{scaffolder => }/util/metrics.ts | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename plugins/scaffolder-backend/src/{scaffolder => }/util/metrics.ts (100%)

diff --git a/plugins/scaffolder-backend/src/scaffolder/util/metrics.ts b/plugins/scaffolder-backend/src/util/metrics.ts
similarity index 100%
rename from plugins/scaffolder-backend/src/scaffolder/util/metrics.ts
rename to plugins/scaffolder-backend/src/util/metrics.ts

From ea14eb62a2f22a236ce99933255cce1b2c2b9e08 Mon Sep 17 00:00:00 2001
From: Spencer Henry <shenry@qualtrics.com>
Date: Thu, 22 Sep 2022 15:37:45 -0600
Subject: [PATCH 04/18] redo changeset

Signed-off-by: Spencer Henry <shenry@qualtrics.com>
---
 .changeset/{silver-games-yell.md => flat-items-perform.md} | 1 -
 1 file changed, 1 deletion(-)
 rename .changeset/{silver-games-yell.md => flat-items-perform.md} (97%)

diff --git a/.changeset/silver-games-yell.md b/.changeset/flat-items-perform.md
similarity index 97%
rename from .changeset/silver-games-yell.md
rename to .changeset/flat-items-perform.md
index 3f9d3ec8bf..3b3dec9013 100644
--- a/.changeset/silver-games-yell.md
+++ b/.changeset/flat-items-perform.md
@@ -1,6 +1,5 @@
 ---
 '@backstage/plugin-scaffolder-backend': patch
-'@backstage/plugin-catalog-backend': patch
 ---
 
 Added a set of default Prometheus metrics around scaffolding. See below for a list of metrics and an explanation of their labels:

From a92cbc2668e3fbed48329b36b6942a106ea68033 Mon Sep 17 00:00:00 2001
From: Spencer Henry <shenry@qualtrics.com>
Date: Tue, 27 Sep 2022 08:12:00 -0600
Subject: [PATCH 05/18] add carat to package.json

Signed-off-by: Spencer Henry <shenry@qualtrics.com>
---
 plugins/scaffolder-backend/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json
index b96be75294..613211b290 100644
--- a/plugins/scaffolder-backend/package.json
+++ b/plugins/scaffolder-backend/package.json
@@ -72,7 +72,7 @@
     "octokit": "^2.0.0",
     "octokit-plugin-create-pull-request": "^3.10.0",
     "p-limit": "^3.1.0",
-    "prom-client": "14.0.1",
+    "prom-client": "^14.0.1",
     "uuid": "^8.2.0",
     "vm2": "^3.9.11",
     "winston": "^3.2.1",

From 92672b15a5cd7e789b870ad43d2973d532871304 Mon Sep 17 00:00:00 2001
From: Spencer Henry <shenry@qualtrics.com>
Date: Tue, 27 Sep 2022 08:16:47 -0600
Subject: [PATCH 06/18] rerun yarn

Signed-off-by: Spencer Henry <shenry@qualtrics.com>
---
 yarn.lock | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 774ad8e3ce..a039eaa65c 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6395,7 +6395,7 @@ __metadata:
     octokit: ^2.0.0
     octokit-plugin-create-pull-request: ^3.10.0
     p-limit: ^3.1.0
-    prom-client: 14.0.1
+    prom-client: ^14.0.1
     supertest: ^6.1.3
     uuid: ^8.2.0
     vm2: ^3.9.11
@@ -32363,15 +32363,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"prom-client@npm:14.0.1":
-  version: 14.0.1
-  resolution: "prom-client@npm:14.0.1"
-  dependencies:
-    tdigest: ^0.1.1
-  checksum: 864c19b7086eda8fae652385bc8b8aeb155f85922e58672d07a64918a603341e120e65e08f9d77ccab546518dc18930284da8743c2aac3c968f626d7063d6bba
-  languageName: node
-  linkType: hard
-
 "prom-client@npm:^14.0.1":
   version: 14.1.0
   resolution: "prom-client@npm:14.1.0"

From c51c1bbc26cb9598977c86ff9a5188b7202e6b0a Mon Sep 17 00:00:00 2001
From: Spencer Henry <shenry@qualtrics.com>
Date: Tue, 27 Sep 2022 09:50:34 -0600
Subject: [PATCH 07/18] md cleanup, add to prom tutorial doc

Signed-off-by: Spencer Henry <shenry@qualtrics.com>
---
 .changeset/flat-items-perform.md             | 10 +++++-----
 contrib/docs/tutorials/prometheus-metrics.md |  6 ++++++
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/.changeset/flat-items-perform.md b/.changeset/flat-items-perform.md
index 3b3dec9013..949814c064 100644
--- a/.changeset/flat-items-perform.md
+++ b/.changeset/flat-items-perform.md
@@ -11,35 +11,35 @@ Added a set of default Prometheus metrics around scaffolding. See below for a li
   - `template`: The entity ref of the scaffolded template
   - `user`: The entity ref of the user that invoked the template run
 
-- scaffolder_task_error_count: a count that track how many task runs error out
+- `scaffolder_task_error_count`: a count that track how many task runs error out
 
   Labels:
 
   - `template`: The entity ref of the scaffolded template
   - `user`: The entity ref of the user that invoked the template run
 
-- scaffolder_task_duration: a histogram which tracks the duration of a task run
+- `scaffolder_task_duration`: a histogram which tracks the duration of a task run
 
   Labels:
 
   - `template`: The entity ref of the scaffolded template
   - `result`: A boolean describing whether the task ran successfully
 
-- scaffolder_step_success_count: a count that tracks each step run
+- `scaffolder_step_success_count`: a count that tracks each step run
 
   Labels:
 
   - `template`: The entity ref of the scaffolded template
   - `step`: The name of the step that was run
 
-- scaffolder_step_error_count: a count that tracks how many steps error out
+- `scaffolder_step_error_count`: a count that tracks how many steps error out
 
   Labels:
 
   - `template`: The entity ref of the scaffolded template
   - `step`: The name of the step that was run
 
-- scaffolder_step_duration: a histogram which tracks the duration of each step run
+- `scaffolder_step_duration`: a histogram which tracks the duration of each step run
 
   Labels:
 
diff --git a/contrib/docs/tutorials/prometheus-metrics.md b/contrib/docs/tutorials/prometheus-metrics.md
index 5b7dc61df3..a46a2286cd 100644
--- a/contrib/docs/tutorials/prometheus-metrics.md
+++ b/contrib/docs/tutorials/prometheus-metrics.md
@@ -106,3 +106,9 @@ There are some custom metrics that have been added to Backstage will be output f
 - `catalog_processing_duration_seconds`: Time spent executing the full processing flow
 - `catalog_processors_duration_seconds`: Time spent executing catalog processors
 - `catalog_processing_queue_delay_seconds`: The amount of delay between being scheduled for processing, and the start of actually being processed
+- `scaffolder_task_success_count`: Tracks successful task runs.
+- `scaffolder_task_error_count`: a count that track how many task runs error out
+- `scaffolder_task_duration`: a histogram which tracks the duration of a task run
+- `scaffolder_step_success_count`: a count that tracks each step run
+- `scaffolder_step_error_count`: a count that tracks how many steps error out
+- `scaffolder_step_duration`: a histogram which tracks the duration of each step run

From 2c45c9d7038efff5ea3a2d20d4a9e8ddc5613046 Mon Sep 17 00:00:00 2001
From: Spencer Henry <shenry@qualtrics.com>
Date: Thu, 29 Sep 2022 09:15:38 -0600
Subject: [PATCH 08/18] kick off E2E tests

Signed-off-by: Spencer Henry <shenry@qualtrics.com>

From 1d4a87eea44f6a5fc5c8982c6c3432eab3fed98c Mon Sep 17 00:00:00 2001
From: Spencer Henry <shenry@qualtrics.com>
Date: Mon, 10 Oct 2022 09:34:34 -0600
Subject: [PATCH 09/18] condense metrics and use a result label

Signed-off-by: Spencer Henry <shenry@qualtrics.com>
---
 .changeset/flat-items-perform.md              | 22 ++++----------
 .../tasks/NunjucksWorkflowRunner.ts           | 30 ++++++++-----------
 2 files changed, 17 insertions(+), 35 deletions(-)

diff --git a/.changeset/flat-items-perform.md b/.changeset/flat-items-perform.md
index 949814c064..ce5b95c7ca 100644
--- a/.changeset/flat-items-perform.md
+++ b/.changeset/flat-items-perform.md
@@ -4,19 +4,13 @@
 
 Added a set of default Prometheus metrics around scaffolding. See below for a list of metrics and an explanation of their labels:
 
-- `scaffolder_task_success_count`: Tracks successful task runs.
-
-  Labels:
-
-  - `template`: The entity ref of the scaffolded template
-  - `user`: The entity ref of the user that invoked the template run
-
-- `scaffolder_task_error_count`: a count that track how many task runs error out
+- `scaffolder_task_count`: Tracks successful task runs.
 
   Labels:
 
   - `template`: The entity ref of the scaffolded template
   - `user`: The entity ref of the user that invoked the template run
+  - `result`: A string describing whether the task ran successfully, errored out, or was skipped
 
 - `scaffolder_task_duration`: a histogram which tracks the duration of a task run
 
@@ -25,19 +19,13 @@ Added a set of default Prometheus metrics around scaffolding. See below for a li
   - `template`: The entity ref of the scaffolded template
   - `result`: A boolean describing whether the task ran successfully
 
-- `scaffolder_step_success_count`: a count that tracks each step run
-
-  Labels:
-
-  - `template`: The entity ref of the scaffolded template
-  - `step`: The name of the step that was run
-
-- `scaffolder_step_error_count`: a count that tracks how many steps error out
+- `scaffolder_step_count`: a count that tracks each step run
 
   Labels:
 
   - `template`: The entity ref of the scaffolded template
   - `step`: The name of the step that was run
+  - `result`: A string describing whether the task ran successfully, errored out, or was skipped
 
 - `scaffolder_step_duration`: a histogram which tracks the duration of each step run
 
@@ -45,6 +33,6 @@ Added a set of default Prometheus metrics around scaffolding. See below for a li
 
   - `template`: The entity ref of the scaffolded template
   - `step`: The name of the step that was run
-  - `result`: A boolean describing whether the task ran successfully
+  - `result`: A string describing whether the task ran successfully, errored out, or was skipped
 
 You can find a guide for running Prometheus metrics here: https://github.com/backstage/backstage/blob/master/contrib/docs/tutorials/prometheus-metrics.md
diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
index f4776bc033..a63bb48694 100644
--- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
+++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
@@ -354,30 +354,20 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
 }
 
 function scaffoldingTracker() {
-  const taskSuccesses = createCounterMetric({
+  const taskCount = createCounterMetric({
     name: 'scaffolder_task_success_count',
     help: 'Count of succesful task runs',
-    labelNames: ['template', 'user'],
-  });
-  const taskErrors = createCounterMetric({
-    name: 'scaffolder_task_error_count',
-    help: 'Count of failed task runs',
-    labelNames: ['template', 'user'],
+    labelNames: ['template', 'user', 'result'],
   });
   const taskDuration = createHistogramMetric({
     name: 'scaffolder_task_duration',
     help: 'Duration of a task run',
     labelNames: ['template', 'result'],
   });
-  const stepSuccesses = createCounterMetric({
+  const stepCount = createCounterMetric({
     name: 'scaffolder_step_success_count',
     help: 'Count of successful step runs',
-    labelNames: ['template', 'step'],
-  });
-  const stepErrors = createCounterMetric({
-    name: 'scaffolder_step_error_count',
-    help: 'Count of failed step runs',
-    labelNames: ['template', 'step'],
+    labelNames: ['template', 'step', 'result'],
   });
   const stepDuration = createHistogramMetric({
     name: 'scaffolder_step_duration',
@@ -405,9 +395,10 @@ function scaffoldingTracker() {
     }
 
     function markSuccessful() {
-      taskSuccesses.inc({
+      taskCount.inc({
         template,
         user,
+        result: 'ok'
       });
       taskTimer({ result: 'ok' });
     }
@@ -417,9 +408,10 @@ function scaffoldingTracker() {
         stepId: step.id,
         status: 'failed',
       });
-      taskErrors.inc({
+      taskCount.inc({
         template,
         user,
+        result: 'failed'
       });
       taskTimer({ result: 'failed' });
     }
@@ -448,17 +440,19 @@ function scaffoldingTracker() {
         stepId: step.id,
         status: 'completed',
       });
-      stepSuccesses.inc({
+      stepCount.inc({
         template,
         step: step.name,
+        result: 'ok'
       });
       stepTimer({ result: 'ok' });
     }
 
     function markFailed() {
-      stepErrors.inc({
+      stepCount.inc({
         template,
         step: step.name,
+        result: 'failed'
       });
       stepTimer({ result: 'failed' });
     }

From 1a9d19dc6d8cb1d06cd1398a500316a688cbeded Mon Sep 17 00:00:00 2001
From: Spencer Henry <shenry@qualtrics.com>
Date: Mon, 10 Oct 2022 09:54:32 -0600
Subject: [PATCH 10/18] linting fixes

Signed-off-by: Spencer Henry <shenry@qualtrics.com>
---
 .changeset/flat-items-perform.md                          | 6 +++---
 .../src/scaffolder/tasks/NunjucksWorkflowRunner.ts        | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.changeset/flat-items-perform.md b/.changeset/flat-items-perform.md
index ce5b95c7ca..46626ac667 100644
--- a/.changeset/flat-items-perform.md
+++ b/.changeset/flat-items-perform.md
@@ -10,7 +10,7 @@ Added a set of default Prometheus metrics around scaffolding. See below for a li
 
   - `template`: The entity ref of the scaffolded template
   - `user`: The entity ref of the user that invoked the template run
-  - `result`: A string describing whether the task ran successfully, errored out, or was skipped
+  - `result`: A string describing whether the task ran successfully, failed, or was skipped
 
 - `scaffolder_task_duration`: a histogram which tracks the duration of a task run
 
@@ -25,7 +25,7 @@ Added a set of default Prometheus metrics around scaffolding. See below for a li
 
   - `template`: The entity ref of the scaffolded template
   - `step`: The name of the step that was run
-  - `result`: A string describing whether the task ran successfully, errored out, or was skipped
+  - `result`: A string describing whether the task ran successfully, failed, or was skipped
 
 - `scaffolder_step_duration`: a histogram which tracks the duration of each step run
 
@@ -33,6 +33,6 @@ Added a set of default Prometheus metrics around scaffolding. See below for a li
 
   - `template`: The entity ref of the scaffolded template
   - `step`: The name of the step that was run
-  - `result`: A string describing whether the task ran successfully, errored out, or was skipped
+  - `result`: A string describing whether the task ran successfully, failed, or was skipped
 
 You can find a guide for running Prometheus metrics here: https://github.com/backstage/backstage/blob/master/contrib/docs/tutorials/prometheus-metrics.md
diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
index a63bb48694..f651be4830 100644
--- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
+++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
@@ -398,7 +398,7 @@ function scaffoldingTracker() {
       taskCount.inc({
         template,
         user,
-        result: 'ok'
+        result: 'ok',
       });
       taskTimer({ result: 'ok' });
     }
@@ -411,7 +411,7 @@ function scaffoldingTracker() {
       taskCount.inc({
         template,
         user,
-        result: 'failed'
+        result: 'failed',
       });
       taskTimer({ result: 'failed' });
     }
@@ -443,7 +443,7 @@ function scaffoldingTracker() {
       stepCount.inc({
         template,
         step: step.name,
-        result: 'ok'
+        result: 'ok',
       });
       stepTimer({ result: 'ok' });
     }
@@ -452,7 +452,7 @@ function scaffoldingTracker() {
       stepCount.inc({
         template,
         step: step.name,
-        result: 'failed'
+        result: 'failed',
       });
       stepTimer({ result: 'failed' });
     }

From f67f7e97e1d607e263adfbaae682f1d5cc7ffbaa Mon Sep 17 00:00:00 2001
From: spencerrichardhenry
 <46569542+spencerrichardhenry@users.noreply.github.com>
Date: Wed, 12 Oct 2022 09:13:19 -0600
Subject: [PATCH 11/18] Update .changeset/flat-items-perform.md

Co-authored-by: Patrik Oldsberg <poldsberg@gmail.com>
Signed-off-by: spencerrichardhenry <46569542+spencerrichardhenry@users.noreply.github.com>
---
 .changeset/flat-items-perform.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.changeset/flat-items-perform.md b/.changeset/flat-items-perform.md
index 46626ac667..4bc846ba7e 100644
--- a/.changeset/flat-items-perform.md
+++ b/.changeset/flat-items-perform.md
@@ -1,5 +1,5 @@
 ---
-'@backstage/plugin-scaffolder-backend': patch
+'@backstage/plugin-scaffolder-backend': minor
 ---
 
 Added a set of default Prometheus metrics around scaffolding. See below for a list of metrics and an explanation of their labels:

From 383dc6e2fa714d89f41c9b204ff526216611e9ab Mon Sep 17 00:00:00 2001
From: Spencer Henry <shenry@qualtrics.com>
Date: Wed, 12 Oct 2022 09:17:21 -0600
Subject: [PATCH 12/18] address mr comments

Signed-off-by: Spencer Henry <shenry@qualtrics.com>
---
 contrib/docs/tutorials/prometheus-metrics.md  |  6 ++---
 .../tasks/NunjucksWorkflowRunner.ts           | 25 +++++++------------
 2 files changed, 11 insertions(+), 20 deletions(-)

diff --git a/contrib/docs/tutorials/prometheus-metrics.md b/contrib/docs/tutorials/prometheus-metrics.md
index a46a2286cd..56989e709c 100644
--- a/contrib/docs/tutorials/prometheus-metrics.md
+++ b/contrib/docs/tutorials/prometheus-metrics.md
@@ -106,9 +106,7 @@ There are some custom metrics that have been added to Backstage will be output f
 - `catalog_processing_duration_seconds`: Time spent executing the full processing flow
 - `catalog_processors_duration_seconds`: Time spent executing catalog processors
 - `catalog_processing_queue_delay_seconds`: The amount of delay between being scheduled for processing, and the start of actually being processed
-- `scaffolder_task_success_count`: Tracks successful task runs.
-- `scaffolder_task_error_count`: a count that track how many task runs error out
+- `scaffolder_task_count`: Tracks successful task runs.
 - `scaffolder_task_duration`: a histogram which tracks the duration of a task run
-- `scaffolder_step_success_count`: a count that tracks each step run
-- `scaffolder_step_error_count`: a count that tracks how many steps error out
+- `scaffolder_step_count`: a count that tracks each step run
 - `scaffolder_step_duration`: a histogram which tracks the duration of each step run
diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
index f651be4830..fa0e9d8402 100644
--- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
+++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
@@ -230,7 +230,6 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
           const { taskLogger, streamLogger } = createStepLogger({ task, step });
 
           if (task.isDryRun) {
-            await taskTrack.skipDryRun(step, action);
             const redactedSecrets = Object.fromEntries(
               Object.entries(task.secrets ?? {}).map(secret => [
                 secret[0],
@@ -258,13 +257,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
               )}`,
             );
             if (!action.supportsDryRun) {
-              task.emitLog(
-                `Skipping because ${action.id} does not support dry-run`,
-                {
-                  stepId: step.id,
-                  status: 'skipped',
-                },
-              );
+              await taskTrack.skipDryRun(step, action);
               const outputSchema = action.schema?.output;
               if (outputSchema) {
                 context.steps[step.id] = {
@@ -336,13 +329,13 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
           await stepTrack.markSuccessful();
         } catch (err) {
           await taskTrack.markFailed(step, err);
-          stepTrack.markFailed();
+          await stepTrack.markFailed();
           throw err;
         }
       }
 
       const output = this.render(task.spec.output, context, renderTemplate);
-      taskTrack.markSuccessful();
+      await taskTrack.markSuccessful();
 
       return { output };
     } finally {
@@ -355,8 +348,8 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
 
 function scaffoldingTracker() {
   const taskCount = createCounterMetric({
-    name: 'scaffolder_task_success_count',
-    help: 'Count of succesful task runs',
+    name: 'scaffolder_task_count',
+    help: 'Count of task runs',
     labelNames: ['template', 'user', 'result'],
   });
   const taskDuration = createHistogramMetric({
@@ -365,8 +358,8 @@ function scaffoldingTracker() {
     labelNames: ['template', 'result'],
   });
   const stepCount = createCounterMetric({
-    name: 'scaffolder_step_success_count',
-    help: 'Count of successful step runs',
+    name: 'scaffolder_step_count',
+    help: 'Count of step runs',
     labelNames: ['template', 'step', 'result'],
   });
   const stepDuration = createHistogramMetric({
@@ -394,7 +387,7 @@ function scaffoldingTracker() {
       });
     }
 
-    function markSuccessful() {
+    async function markSuccessful() {
       taskCount.inc({
         template,
         user,
@@ -448,7 +441,7 @@ function scaffoldingTracker() {
       stepTimer({ result: 'ok' });
     }
 
-    function markFailed() {
+    async function markFailed() {
       stepCount.inc({
         template,
         step: step.name,

From 2499dcb7b123b15c64c4129270737d3582351ec3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= <freben@gmail.com>
Date: Thu, 22 Sep 2022 14:09:19 +0200
Subject: [PATCH 13/18] no longer transpile tsx files in backend packages
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Fredrik Adelöw <freben@gmail.com>
---
 .changeset/swift-phones-cheat.md       | 12 ++++++++++++
 packages/cli/src/lib/bundler/config.ts |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)
 create mode 100644 .changeset/swift-phones-cheat.md

diff --git a/.changeset/swift-phones-cheat.md b/.changeset/swift-phones-cheat.md
new file mode 100644
index 0000000000..97697ad916
--- /dev/null
+++ b/.changeset/swift-phones-cheat.md
@@ -0,0 +1,12 @@
+---
+'@backstage/cli': minor
+---
+
+Removed `tsx` and `jsx` as supported extensions in backend packages. For most
+repos, this will not have any effect. But if you inadvertently had added some
+`tsx`/`jsx` files to your backend package, you may now start to see `code: 'MODULE_NOT_FOUND'` errors when launching the backend locally. The reason for
+this is that the offending files get ignored during transpilation. Hence, the
+importing file can no longer find anything to import.
+
+The fix is to rename any `.tsx` files in your backend packages to `.ts` instead,
+or `.jsx` to `.js`.
diff --git a/packages/cli/src/lib/bundler/config.ts b/packages/cli/src/lib/bundler/config.ts
index 0d4649fa3f..1974153f02 100644
--- a/packages/cli/src/lib/bundler/config.ts
+++ b/packages/cli/src/lib/bundler/config.ts
@@ -267,7 +267,7 @@ export async function createBackendConfig(
       paths.targetRunFile ? paths.targetRunFile : paths.targetEntry,
     ],
     resolve: {
-      extensions: ['.ts', '.tsx', '.mjs', '.js', '.jsx', '.json'],
+      extensions: ['.ts', '.mjs', '.js', '.json'],
       mainFields: ['main'],
       modules: [paths.rootNodeModules, ...moduleDirs],
       plugins: [

From 19ff4836a5af9c64d900330c8f2a7c0e623acb3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= <freben@gmail.com>
Date: Thu, 22 Sep 2022 17:03:20 +0200
Subject: [PATCH 14/18] nerf
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Fredrik Adelöw <freben@gmail.com>
---
 .changeset/swift-phones-cheat.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.changeset/swift-phones-cheat.md b/.changeset/swift-phones-cheat.md
index 97697ad916..6d1653caa3 100644
--- a/.changeset/swift-phones-cheat.md
+++ b/.changeset/swift-phones-cheat.md
@@ -1,5 +1,5 @@
 ---
-'@backstage/cli': minor
+'@backstage/cli': patch
 ---
 
 Removed `tsx` and `jsx` as supported extensions in backend packages. For most

From c735a18c0db0127dd67ba6a1b781fe43773e64b0 Mon Sep 17 00:00:00 2001
From: Spencer Henry <shenry@qualtrics.com>
Date: Thu, 29 Sep 2022 09:15:38 -0600
Subject: [PATCH 15/18] kick off E2E tests

Signed-off-by: Spencer Henry <shenry@qualtrics.com>

From aff603e4bdfbfc5f6aba60d28d02ab59ba2eb431 Mon Sep 17 00:00:00 2001
From: spencerrichardhenry
 <46569542+spencerrichardhenry@users.noreply.github.com>
Date: Tue, 18 Oct 2022 09:22:43 -0600
Subject: [PATCH 16/18] Update
 plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts

Co-authored-by: Patrik Oldsberg <poldsberg@gmail.com>
Signed-off-by: spencerrichardhenry <46569542+spencerrichardhenry@users.noreply.github.com>
---
 .../src/scaffolder/tasks/NunjucksWorkflowRunner.ts              | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
index fa0e9d8402..4f3ca032bc 100644
--- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
+++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
@@ -452,7 +452,7 @@ function scaffoldingTracker() {
 
     async function skipFalsy() {
       await task.emitLog(
-        `Skipping step ${step.id} because it's if condition was false`,
+        `Skipping step ${step.id} because its if condition was false`,
         { stepId: step.id, status: 'skipped' },
       );
       stepTimer({ result: 'skipped' });

From 51d891bcf7afec67d3e91cc02d6802253638f640 Mon Sep 17 00:00:00 2001
From: Spencer Henry <shenry@qualtrics.com>
Date: Tue, 18 Oct 2022 09:26:44 -0600
Subject: [PATCH 17/18] remove random commit items?

Signed-off-by: Spencer Henry <shenry@qualtrics.com>
---
 .changeset/swift-phones-cheat.md                     | 12 ------------
 .../src/tasks/PluginTaskSchedulerImpl.test.ts        |  2 +-
 packages/cli/src/lib/bundler/config.ts               |  2 +-
 3 files changed, 2 insertions(+), 14 deletions(-)
 delete mode 100644 .changeset/swift-phones-cheat.md

diff --git a/.changeset/swift-phones-cheat.md b/.changeset/swift-phones-cheat.md
deleted file mode 100644
index 6d1653caa3..0000000000
--- a/.changeset/swift-phones-cheat.md
+++ /dev/null
@@ -1,12 +0,0 @@
----
-'@backstage/cli': patch
----
-
-Removed `tsx` and `jsx` as supported extensions in backend packages. For most
-repos, this will not have any effect. But if you inadvertently had added some
-`tsx`/`jsx` files to your backend package, you may now start to see `code: 'MODULE_NOT_FOUND'` errors when launching the backend locally. The reason for
-this is that the offending files get ignored during transpilation. Hence, the
-importing file can no longer find anything to import.
-
-The fix is to rename any `.tsx` files in your backend packages to `.ts` instead,
-or `.jsx` to `.js`.
diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts
index d61948a929..f6177b57f7 100644
--- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts
+++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts
@@ -45,7 +45,7 @@ describe('PluginTaskManagerImpl', () => {
     );
 
     jest.useFakeTimers();
-  }, 60_000);
+  }, 30_000);
 
   async function init(databaseId: TestDatabaseId) {
     const knex = await databases.init(databaseId);
diff --git a/packages/cli/src/lib/bundler/config.ts b/packages/cli/src/lib/bundler/config.ts
index 1974153f02..7ce9499f45 100644
--- a/packages/cli/src/lib/bundler/config.ts
+++ b/packages/cli/src/lib/bundler/config.ts
@@ -267,7 +267,7 @@ export async function createBackendConfig(
       paths.targetRunFile ? paths.targetRunFile : paths.targetEntry,
     ],
     resolve: {
-      extensions: ['.ts', '.mjs', '.js', '.json'],
+      extensions: ['.ts', '.tsx', '.js', '.jsx', '.json'],
       mainFields: ['main'],
       modules: [paths.rootNodeModules, ...moduleDirs],
       plugins: [

From 5698213eb451ede9e29d1580f4781b88103e6fac Mon Sep 17 00:00:00 2001
From: Spencer Henry <shenry@qualtrics.com>
Date: Thu, 29 Sep 2022 09:15:38 -0600
Subject: [PATCH 18/18] kick off E2E tests

Signed-off-by: Spencer Henry <shenry@qualtrics.com>