Merge pull request #34160 from backstage/freben/catalog-metrics-cache

fix(catalog-backend): cache and cheapen catalog_entities_count metric

.changeset/catalog-metrics-cache.md

@@ -0,0 +1,9 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Improved the performance of the `catalog_entities_count` metric.
+
+The legacy Prometheus and OpenTelemetry observable gauges previously each ran their own copy of the per-kind count query against the `search` table on every metrics scrape. On large catalogs this could pile up faster than the queries completed, contending for buffers and stalling the database.
+
+The two callbacks now share a single query result with a short in-process TTL cache, and the underlying query reads from `final_entities` instead of `search`, avoiding the bitmap heap scans that dominated the previous form. The emitted labels and values are unchanged.

plugins/catalog-backend/src/database/metrics.test.ts

@@ -0,0 +1,162 @@
+/*
+ * Copyright 2026 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
+import { Knex } from 'knex';
+import { randomUUID as uuid } from 'node:crypto';
+import { applyDatabaseMigrations } from './migrations';
+import { DbFinalEntitiesRow, DbRefreshStateRow } from './tables';
+import { createEntitiesCountByKind, queryEntitiesCountByKind } from './metrics';
+
+jest.setTimeout(60_000);
+
+describe('metrics', () => {
+  const databases = TestDatabases.create();
+
+  async function createDatabase(databaseId: TestDatabaseId) {
+    const knex = await databases.init(databaseId);
+    await applyDatabaseMigrations(knex);
+    return knex;
+  }
+
+  async function insertEntity(
+    knex: Knex,
+    options: { entityRef: string; finalEntity: string | null },
+  ): Promise<void> {
+    const entityId = uuid();
+    await knex<DbRefreshStateRow>('refresh_state').insert({
+      entity_id: entityId,
+      entity_ref: options.entityRef,
+      unprocessed_entity: '{}',
+      errors: '[]',
+      next_update_at: '2021-04-01 13:37:00',
+      last_discovery_at: '2021-04-01 13:37:00',
+    });
+    await knex<DbFinalEntitiesRow>('final_entities').insert({
+      entity_id: entityId,
+      entity_ref: options.entityRef,
+      hash: 'h',
+      final_entity: options.finalEntity ?? undefined,
+      last_updated_at: '2021-04-01 13:37:00',
+    });
+  }
+
+  describe('queryEntitiesCountByKind', () => {
+    it.each(databases.eachSupportedId())(
+      'counts entities grouped by the kind in entity_ref, %p',
+      async databaseId => {
+        const knex = await createDatabase(databaseId);
+
+        await insertEntity(knex, {
+          entityRef: 'component:default/svc-a',
+          finalEntity: '{"kind":"Component"}',
+        });
+        await insertEntity(knex, {
+          entityRef: 'component:default/svc-b',
+          finalEntity: '{"kind":"Component"}',
+        });
+        await insertEntity(knex, {
+          entityRef: 'api:default/api-a',
+          finalEntity: '{"kind":"API"}',
+        });
+        await insertEntity(knex, {
+          entityRef: 'system:other/sys-a',
+          finalEntity: '{"kind":"System"}',
+        });
+        // Not yet stitched -- must be excluded from the count
+        await insertEntity(knex, {
+          entityRef: 'component:default/pending',
+          finalEntity: null,
+        });
+
+        const result = await queryEntitiesCountByKind(knex);
+
+        expect(Object.fromEntries(result)).toEqual({
+          component: 2,
+          api: 1,
+          system: 1,
+        });
+      },
+    );
+  });
+
+  describe('createEntitiesCountByKind', () => {
+    it.each(databases.eachSupportedId())(
+      'serves cached results within the TTL and refreshes after, %p',
+      async databaseId => {
+        const knex = await createDatabase(databaseId);
+        const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 });
+
+        await insertEntity(knex, {
+          entityRef: 'component:default/one',
+          finalEntity: '{}',
+        });
+
+        const first = await getCount();
+        expect(Object.fromEntries(first)).toEqual({ component: 1 });
+
+        // A change made within the TTL window must not be visible yet.
+        await insertEntity(knex, {
+          entityRef: 'component:default/two',
+          finalEntity: '{}',
+        });
+        const cached = await getCount();
+        expect(Object.fromEntries(cached)).toEqual({ component: 1 });
+
+        // After the TTL elapses the next call hits the database again.
+        await new Promise(resolve => setTimeout(resolve, 80));
+        const refreshed = await getCount();
+        expect(Object.fromEntries(refreshed)).toEqual({ component: 2 });
+      },
+    );
+
+    it.each(databases.eachSupportedId())(
+      'coalesces overlapping callers into a single underlying query, %p',
+      async databaseId => {
+        const knex = await createDatabase(databaseId);
+        const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 });
+
+        await insertEntity(knex, {
+          entityRef: 'component:default/one',
+          finalEntity: '{}',
+        });
+
+        const finalEntitiesQueries: string[] = [];
+        knex.on('query', (q: { sql: string }) => {
+          if (
+            /from\s+["`]?final_entities["`]?/i.test(q.sql) &&
+            /^\s*select/i.test(q.sql)
+          ) {
+            finalEntitiesQueries.push(q.sql);
+          }
+        });
+
+        // Five concurrent callers should result in one query, not five.
+        const results = await Promise.all([
+          getCount(),
+          getCount(),
+          getCount(),
+          getCount(),
+          getCount(),
+        ]);
+        expect(finalEntitiesQueries).toHaveLength(1);
+        for (const r of results) {
+          expect(Object.fromEntries(r)).toEqual({ component: 1 });
+        }
+      },
+    );
+  });
+});

plugins/catalog-backend/src/database/metrics.ts

@@ -16,12 +16,102 @@
 
 import { Knex } from 'knex';
 import { createGaugeMetric } from '../util/metrics';
-import { DbRelationsRow, DbLocationsRow, DbSearchRow } from './tables';
+import { DbRelationsRow, DbLocationsRow } from './tables';
 import { MetricsService } from '@backstage/backend-plugin-api/alpha';
 
+const ENTITIES_COUNT_TTL_MS = 30_000;
+
+/**
+ * Returns a function that produces a Map of entity kind -> count, with
+ * a single-flight cache that coalesces overlapping calls.
+ *
+ * The OpenTelemetry observable gauge and the legacy Prometheus gauge are
+ * both registered to emit the same `catalog_entities_count` series, and
+ * both fire on every metrics scrape. Without coalescing, that means two
+ * identical heavy queries per scrape per pod, which can pile up against
+ * the database faster than they complete.
+ *
+ * Concurrent callers share the same in-flight promise, so a query is
+ * never overlapped by another instance of itself. The TTL is the
+ * minimum age at which the next caller is allowed to start a fresh
+ * query; if a query is still running when the TTL would have elapsed,
+ * waiting callers continue to share that one rather than starting a
+ * duplicate.
+ *
+ * @internal exported for testing
+ */
+export function createEntitiesCountByKind(
+  knex: Knex,
+  options?: { ttlMs?: number },
+): () => Promise<Map<string, number>> {
+  const ttlMs = options?.ttlMs ?? ENTITIES_COUNT_TTL_MS;
+  let inflight: Promise<Map<string, number>> | undefined;
+  let cached: { at: number; data: Map<string, number> } | undefined;
+  return () => {
+    if (inflight) {
+      return inflight;
+    }
+    if (cached && Date.now() - cached.at < ttlMs) {
+      return Promise.resolve(cached.data);
+    }
+    inflight = (async () => {
+      try {
+        const data = await queryEntitiesCountByKind(knex);
+        cached = { at: Date.now(), data };
+        return data;
+      } finally {
+        inflight = undefined;
+      }
+    })();
+    return inflight;
+  };
+}
+
+/**
+ * Reads kind counts straight from `final_entities` (one row per entity)
+ * rather than from `search` (one row per entity per indexed key, often
+ * 20-30x larger). The kind is parsed out of `entity_ref`, which is the
+ * canonical lowercased `kind:namespace/name` form -- producing the same
+ * lowercased labels the previous search-based query did.
+ *
+ * Rows where `final_entity` is null are excluded. They represent entities
+ * that haven't been stitched yet, or tombstones for in-progress deletions.
+ * Counting them would over-report by including entities the rest of the
+ * catalog API treats as not present.
+ *
+ * @internal exported for testing
+ */
+export async function queryEntitiesCountByKind(
+  knex: Knex,
+): Promise<Map<string, number>> {
+  const kindExpr = entityRefKindExpression(knex);
+
+  const rows: { kind: string; count: string | number }[] = await knex(
+    'final_entities',
+  )
+    .whereNotNull('final_entity')
+    .select({ kind: kindExpr, count: knex.raw('count(*)') })
+    .groupBy(kindExpr);
+
+  return new Map(rows.map(row => [String(row.kind), Number(row.count)]));
+}
+
+function entityRefKindExpression(knex: Knex): Knex.Raw {
+  const client = knex.client.config.client as string;
+  if (client.includes('pg')) {
+    return knex.raw(`split_part(entity_ref, ':', 1)`);
+  }
+  if (client.includes('mysql')) {
+    return knex.raw(`substring_index(entity_ref, ':', 1)`);
+  }
+  // sqlite (better-sqlite3, sqlite3)
+  return knex.raw(`substr(entity_ref, 1, instr(entity_ref, ':') - 1)`);
+}
+
 export function initDatabaseMetrics(knex: Knex, metrics: MetricsService) {
   const seenProm = new Set<string>();
   const seen = new Set<string>();
+  const getEntitiesCountByKind = createEntitiesCountByKind(knex);
 
   return {
     entities_count_prom: createGaugeMetric({
@@ -29,24 +119,20 @@ export function initDatabaseMetrics(knex: Knex, metrics: MetricsService) {
       help: 'Total amount of entities in the catalog. DEPRECATED: Please use opentelemetry metrics instead.',
       labelNames: ['kind'],
       async collect() {
-        const results = await knex<DbSearchRow>('search')
-          .where('key', '=', 'kind')
-          .whereNotNull('value')
-          .select({ kind: 'value', count: knex.raw('count(*)') })
-          .groupBy('value');
+        const results = await getEntitiesCountByKind();
 
-        results.forEach(({ kind, count }) => {
+        for (const [kind, count] of results) {
           seenProm.add(kind);
-          this.set({ kind }, Number(count));
-        });
+          this.set({ kind }, count);
+        }
 
         // Set all the entities that were not seenProm to 0 and delete them from the seenProm set.
-        seenProm.forEach(kind => {
-          if (!results.some(r => r.kind === kind)) {
+        for (const kind of seenProm) {
+          if (!results.has(kind)) {
             this.set({ kind }, 0);
             seenProm.delete(kind);
           }
-        });
+        }
       },
     }),
     registered_locations_prom: createGaugeMetric({
@@ -74,24 +160,20 @@ export function initDatabaseMetrics(knex: Knex, metrics: MetricsService) {
         description: 'Total amount of entities in the catalog',
       })
       .addCallback(async gauge => {
-        const results = await knex<DbSearchRow>('search')
-          .where('key', '=', 'kind')
-          .whereNotNull('value')
-          .select({ kind: 'value', count: knex.raw('count(*)') })
-          .groupBy('value');
+        const results = await getEntitiesCountByKind();
 
-        results.forEach(({ kind, count }) => {
+        for (const [kind, count] of results) {
           seen.add(kind);
-          gauge.observe(Number(count), { kind });
-        });
+          gauge.observe(count, { kind });
+        }
 
         // Set all the entities that were not seen to 0 and delete them from the seen set.
-        seen.forEach(kind => {
-          if (!results.some(r => r.kind === kind)) {
+        for (const kind of seen) {
+          if (!results.has(kind)) {
             gauge.observe(0, { kind });
             seen.delete(kind);
           }
-        });
+        }
       }),
     registered_locations: metrics
       .createObservableGauge('catalog_registered_locations_count', {