Compute some metrics using search table facet aggregations

Signed-off-by: Fredrik Adelöw <freben@gmail.com>

.changeset/young-carrots-collect.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Compute some metrics using search table facet aggregations instead of reading the full refresh state

plugins/catalog-backend/src/database/metrics.ts

@@ -16,9 +16,8 @@
 
 import { Knex } from 'knex';
 import { createGaugeMetric } from '../util/metrics';
-import { DbRefreshStateRow, DbRelationsRow, DbLocationsRow } from './tables';
+import { DbRelationsRow, DbLocationsRow, DbSearchRow } from './tables';
 import { metrics } from '@opentelemetry/api';
-import { parseEntityRef } from '@backstage/catalog-model';
 
 export function initDatabaseMetrics(knex: Knex) {
   const seenProm = new Set<string>();
@@ -30,23 +29,22 @@ export function initDatabaseMetrics(knex: Knex) {
       help: 'Total amount of entities in the catalog. DEPRECATED: Please use opentelemetry metrics instead.',
       labelNames: ['kind'],
       async collect() {
-        const result = await knex<DbRefreshStateRow>('refresh_state').select(
-          'entity_ref',
-        );
-        const results = result
-          .map(row => row.entity_ref.split(':')[0])
-          .reduce((acc, e) => acc.set(e, (acc.get(e) || 0) + 1), new Map());
+        const results = await knex<DbSearchRow>('search')
+          .where('key', '=', 'kind')
+          .whereNotNull('original_value')
+          .select({ kind: 'original_value', count: knex.raw('count(*)') })
+          .groupBy('original_value');
 
-        results.forEach((value, key) => {
-          seenProm.add(key);
-          this.set({ kind: key }, value);
+        results.forEach(({ kind, count }) => {
+          seenProm.add(kind);
+          this.set({ kind }, Number(count));
         });
 
         // Set all the entities that were not seenProm to 0 and delete them from the seenProm set.
-        seenProm.forEach(key => {
-          if (!results.has(key)) {
-            this.set({ kind: key }, 0);
-            seenProm.delete(key);
+        seenProm.forEach(kind => {
+          if (!results.some(r => r.kind === kind)) {
+            this.set({ kind }, 0);
+            seenProm.delete(kind);
           }
         });
       },
@@ -76,23 +74,22 @@ export function initDatabaseMetrics(knex: Knex) {
         description: 'Total amount of entities in the catalog',
       })
       .addCallback(async gauge => {
-        const result = await knex<DbRefreshStateRow>('refresh_state').select(
-          'entity_ref',
-        );
-        const results = result
-          .map(row => parseEntityRef(row.entity_ref).kind)
-          .reduce((acc, e) => acc.set(e, (acc.get(e) || 0) + 1), new Map());
+        const results = await knex<DbSearchRow>('search')
+          .where('key', '=', 'kind')
+          .whereNotNull('original_value')
+          .select({ kind: 'original_value', count: knex.raw('count(*)') })
+          .groupBy('original_value');
 
-        results.forEach((value, key) => {
-          seen.add(key);
-          gauge.observe(value, { kind: key });
+        results.forEach(({ kind, count }) => {
+          seen.add(kind);
+          gauge.observe(Number(count), { kind });
         });
 
         // Set all the entities that were not seen to 0 and delete them from the seen set.
-        seen.forEach(key => {
-          if (!results.has(key)) {
-            gauge.observe(0, { kind: key });
-            seen.delete(key);
+        seen.forEach(kind => {
+          if (!results.some(r => r.kind === kind)) {
+            gauge.observe(0, { kind });
+            seen.delete(kind);
           }
         });
       }),