From ccbad9d892a23d4425e6e994145653591fb73a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 8 May 2026 11:48:58 +0200 Subject: [PATCH 1/2] fix(catalog-backend): cache and cheapen catalog_entities_count metric MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy Prometheus and OpenTelemetry observable gauges previously each ran the per-kind count query against the search table on every metrics scrape. With multiple pods and short scrape intervals, identical sequential scans piled up faster than they completed, contending for buffers in the database. Extract a shared helper that wraps a 30-second TTL cache around a single query, and have both gauges read from it. The query itself moves from the (large) search table to final_entities, parsing kind out of entity_ref via per-engine substring functions. The emitted labels and values are unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Fredrik Adelöw --- .changeset/catalog-metrics-cache.md | 9 ++ .../src/database/metrics.test.ts | 126 ++++++++++++++++++ .../catalog-backend/src/database/metrics.ts | 111 +++++++++++---- 3 files changed, 223 insertions(+), 23 deletions(-) create mode 100644 .changeset/catalog-metrics-cache.md create mode 100644 plugins/catalog-backend/src/database/metrics.test.ts diff --git a/.changeset/catalog-metrics-cache.md b/.changeset/catalog-metrics-cache.md new file mode 100644 index 0000000000..bbe427bcea --- /dev/null +++ b/.changeset/catalog-metrics-cache.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Improved the performance of the `catalog_entities_count` metric. + +The legacy Prometheus and OpenTelemetry observable gauges previously each ran their own copy of the per-kind count query against the `search` table on every metrics scrape. On large catalogs this could pile up faster than the queries completed, contending for buffers and stalling the database. + +The two callbacks now share a single query result with a short in-process TTL cache, and the underlying query reads from `final_entities` instead of `search`, avoiding the bitmap heap scans that dominated the previous form. The emitted labels and values are unchanged. diff --git a/plugins/catalog-backend/src/database/metrics.test.ts b/plugins/catalog-backend/src/database/metrics.test.ts new file mode 100644 index 0000000000..33b1379a3a --- /dev/null +++ b/plugins/catalog-backend/src/database/metrics.test.ts @@ -0,0 +1,126 @@ +/* + * Copyright 2026 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; +import { Knex } from 'knex'; +import { randomUUID as uuid } from 'node:crypto'; +import { applyDatabaseMigrations } from './migrations'; +import { DbFinalEntitiesRow, DbRefreshStateRow } from './tables'; +import { createEntitiesCountByKind, queryEntitiesCountByKind } from './metrics'; + +jest.setTimeout(60_000); + +describe('metrics', () => { + const databases = TestDatabases.create(); + + async function createDatabase(databaseId: TestDatabaseId) { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + return knex; + } + + async function insertEntity( + knex: Knex, + options: { entityRef: string; finalEntity: string | null }, + ): Promise { + const entityId = uuid(); + await knex('refresh_state').insert({ + entity_id: entityId, + entity_ref: options.entityRef, + unprocessed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + await knex('final_entities').insert({ + entity_id: entityId, + entity_ref: options.entityRef, + hash: 'h', + final_entity: options.finalEntity ?? undefined, + last_updated_at: '2021-04-01 13:37:00', + }); + } + + describe('queryEntitiesCountByKind', () => { + it.each(databases.eachSupportedId())( + 'counts entities grouped by the kind in entity_ref, %p', + async databaseId => { + const knex = await createDatabase(databaseId); + + await insertEntity(knex, { + entityRef: 'component:default/svc-a', + finalEntity: '{"kind":"Component"}', + }); + await insertEntity(knex, { + entityRef: 'component:default/svc-b', + finalEntity: '{"kind":"Component"}', + }); + await insertEntity(knex, { + entityRef: 'api:default/api-a', + finalEntity: '{"kind":"API"}', + }); + await insertEntity(knex, { + entityRef: 'system:other/sys-a', + finalEntity: '{"kind":"System"}', + }); + // Not yet stitched -- must be excluded from the count + await insertEntity(knex, { + entityRef: 'component:default/pending', + finalEntity: null, + }); + + const result = await queryEntitiesCountByKind(knex); + + expect(Object.fromEntries(result)).toEqual({ + component: 2, + api: 1, + system: 1, + }); + }, + ); + }); + + describe('createEntitiesCountByKind', () => { + it.each(databases.eachSupportedId())( + 'serves cached results within the TTL and refreshes after, %p', + async databaseId => { + const knex = await createDatabase(databaseId); + const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 }); + + await insertEntity(knex, { + entityRef: 'component:default/one', + finalEntity: '{}', + }); + + const first = await getCount(); + expect(Object.fromEntries(first)).toEqual({ component: 1 }); + + // A change made within the TTL window must not be visible yet. + await insertEntity(knex, { + entityRef: 'component:default/two', + finalEntity: '{}', + }); + const cached = await getCount(); + expect(Object.fromEntries(cached)).toEqual({ component: 1 }); + + // After the TTL elapses the next call hits the database again. + await new Promise(resolve => setTimeout(resolve, 80)); + const refreshed = await getCount(); + expect(Object.fromEntries(refreshed)).toEqual({ component: 2 }); + }, + ); + }); +}); diff --git a/plugins/catalog-backend/src/database/metrics.ts b/plugins/catalog-backend/src/database/metrics.ts index f6a1464b3a..6b45a46dde 100644 --- a/plugins/catalog-backend/src/database/metrics.ts +++ b/plugins/catalog-backend/src/database/metrics.ts @@ -16,12 +16,85 @@ import { Knex } from 'knex'; import { createGaugeMetric } from '../util/metrics'; -import { DbRelationsRow, DbLocationsRow, DbSearchRow } from './tables'; +import { DbRelationsRow, DbLocationsRow } from './tables'; import { MetricsService } from '@backstage/backend-plugin-api/alpha'; +const ENTITIES_COUNT_TTL_MS = 30_000; + +/** + * Returns a function that produces a Map of entity kind -> count, using + * a short in-process TTL cache. + * + * The OpenTelemetry observable gauge and the legacy Prometheus gauge are + * both registered to emit the same `catalog_entities_count` series, and + * both fire on every metrics scrape. Without the cache, that means two + * identical heavy queries per scrape per pod, which can pile up against + * the database faster than they complete. + * + * @internal exported for testing + */ +export function createEntitiesCountByKind( + knex: Knex, + options?: { ttlMs?: number }, +): () => Promise> { + const ttlMs = options?.ttlMs ?? ENTITIES_COUNT_TTL_MS; + let cache: { at: number; data: Map } | undefined; + return async () => { + const now = Date.now(); + if (cache && now - cache.at < ttlMs) { + return cache.data; + } + const data = await queryEntitiesCountByKind(knex); + cache = { at: now, data }; + return data; + }; +} + +/** + * Reads kind counts straight from `final_entities` (one row per entity) + * rather than from `search` (one row per entity per indexed key, often + * 20-30x larger). The kind is parsed out of `entity_ref`, which is the + * canonical lowercased `kind:namespace/name` form -- producing the same + * lowercased labels the previous search-based query did. + * + * Rows where `final_entity` is null are excluded. They represent entities + * that haven't been stitched yet, or tombstones for in-progress deletions. + * Counting them would over-report by including entities the rest of the + * catalog API treats as not present. + * + * @internal exported for testing + */ +export async function queryEntitiesCountByKind( + knex: Knex, +): Promise> { + const kindExpr = entityRefKindExpression(knex); + + const rows: { kind: string; count: string | number }[] = await knex( + 'final_entities', + ) + .whereNotNull('final_entity') + .select({ kind: kindExpr, count: knex.raw('count(*)') }) + .groupBy(kindExpr); + + return new Map(rows.map(row => [String(row.kind), Number(row.count)])); +} + +function entityRefKindExpression(knex: Knex): Knex.Raw { + const client = knex.client.config.client as string; + if (client.includes('pg')) { + return knex.raw(`split_part(entity_ref, ':', 1)`); + } + if (client.includes('mysql')) { + return knex.raw(`substring_index(entity_ref, ':', 1)`); + } + // sqlite (better-sqlite3, sqlite3) + return knex.raw(`substr(entity_ref, 1, instr(entity_ref, ':') - 1)`); +} + export function initDatabaseMetrics(knex: Knex, metrics: MetricsService) { const seenProm = new Set(); const seen = new Set(); + const getEntitiesCountByKind = createEntitiesCountByKind(knex); return { entities_count_prom: createGaugeMetric({ @@ -29,24 +102,20 @@ export function initDatabaseMetrics(knex: Knex, metrics: MetricsService) { help: 'Total amount of entities in the catalog. DEPRECATED: Please use opentelemetry metrics instead.', labelNames: ['kind'], async collect() { - const results = await knex('search') - .where('key', '=', 'kind') - .whereNotNull('value') - .select({ kind: 'value', count: knex.raw('count(*)') }) - .groupBy('value'); + const results = await getEntitiesCountByKind(); - results.forEach(({ kind, count }) => { + for (const [kind, count] of results) { seenProm.add(kind); - this.set({ kind }, Number(count)); - }); + this.set({ kind }, count); + } // Set all the entities that were not seenProm to 0 and delete them from the seenProm set. - seenProm.forEach(kind => { - if (!results.some(r => r.kind === kind)) { + for (const kind of seenProm) { + if (!results.has(kind)) { this.set({ kind }, 0); seenProm.delete(kind); } - }); + } }, }), registered_locations_prom: createGaugeMetric({ @@ -74,24 +143,20 @@ export function initDatabaseMetrics(knex: Knex, metrics: MetricsService) { description: 'Total amount of entities in the catalog', }) .addCallback(async gauge => { - const results = await knex('search') - .where('key', '=', 'kind') - .whereNotNull('value') - .select({ kind: 'value', count: knex.raw('count(*)') }) - .groupBy('value'); + const results = await getEntitiesCountByKind(); - results.forEach(({ kind, count }) => { + for (const [kind, count] of results) { seen.add(kind); - gauge.observe(Number(count), { kind }); - }); + gauge.observe(count, { kind }); + } // Set all the entities that were not seen to 0 and delete them from the seen set. - seen.forEach(kind => { - if (!results.some(r => r.kind === kind)) { + for (const kind of seen) { + if (!results.has(kind)) { gauge.observe(0, { kind }); seen.delete(kind); } - }); + } }), registered_locations: metrics .createObservableGauge('catalog_registered_locations_count', { From 8f7f591c46d2890628463ac5053d6c5834dddbe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 8 May 2026 11:55:26 +0200 Subject: [PATCH 2/2] coalesce overlapping callers via single-flight in-flight promise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hold a shared promise rather than just a resolved value. Concurrent callers awaiting a fresh count get the same in-flight promise back, so the underlying query is never overlapped by a duplicate. The TTL is now the minimum gap between the resolution of one query and the start of the next, rather than a hard bound on cache age. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Fredrik Adelöw --- .../src/database/metrics.test.ts | 36 +++++++++++++++++ .../catalog-backend/src/database/metrics.ts | 39 +++++++++++++------ 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/plugins/catalog-backend/src/database/metrics.test.ts b/plugins/catalog-backend/src/database/metrics.test.ts index 33b1379a3a..74613dcc91 100644 --- a/plugins/catalog-backend/src/database/metrics.test.ts +++ b/plugins/catalog-backend/src/database/metrics.test.ts @@ -122,5 +122,41 @@ describe('metrics', () => { expect(Object.fromEntries(refreshed)).toEqual({ component: 2 }); }, ); + + it.each(databases.eachSupportedId())( + 'coalesces overlapping callers into a single underlying query, %p', + async databaseId => { + const knex = await createDatabase(databaseId); + const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 }); + + await insertEntity(knex, { + entityRef: 'component:default/one', + finalEntity: '{}', + }); + + const finalEntitiesQueries: string[] = []; + knex.on('query', (q: { sql: string }) => { + if ( + /from\s+["`]?final_entities["`]?/i.test(q.sql) && + /^\s*select/i.test(q.sql) + ) { + finalEntitiesQueries.push(q.sql); + } + }); + + // Five concurrent callers should result in one query, not five. + const results = await Promise.all([ + getCount(), + getCount(), + getCount(), + getCount(), + getCount(), + ]); + expect(finalEntitiesQueries).toHaveLength(1); + for (const r of results) { + expect(Object.fromEntries(r)).toEqual({ component: 1 }); + } + }, + ); }); }); diff --git a/plugins/catalog-backend/src/database/metrics.ts b/plugins/catalog-backend/src/database/metrics.ts index 6b45a46dde..40a3c62b7a 100644 --- a/plugins/catalog-backend/src/database/metrics.ts +++ b/plugins/catalog-backend/src/database/metrics.ts @@ -22,15 +22,22 @@ import { MetricsService } from '@backstage/backend-plugin-api/alpha'; const ENTITIES_COUNT_TTL_MS = 30_000; /** - * Returns a function that produces a Map of entity kind -> count, using - * a short in-process TTL cache. + * Returns a function that produces a Map of entity kind -> count, with + * a single-flight cache that coalesces overlapping calls. * * The OpenTelemetry observable gauge and the legacy Prometheus gauge are * both registered to emit the same `catalog_entities_count` series, and - * both fire on every metrics scrape. Without the cache, that means two + * both fire on every metrics scrape. Without coalescing, that means two * identical heavy queries per scrape per pod, which can pile up against * the database faster than they complete. * + * Concurrent callers share the same in-flight promise, so a query is + * never overlapped by another instance of itself. The TTL is the + * minimum age at which the next caller is allowed to start a fresh + * query; if a query is still running when the TTL would have elapsed, + * waiting callers continue to share that one rather than starting a + * duplicate. + * * @internal exported for testing */ export function createEntitiesCountByKind( @@ -38,15 +45,25 @@ export function createEntitiesCountByKind( options?: { ttlMs?: number }, ): () => Promise> { const ttlMs = options?.ttlMs ?? ENTITIES_COUNT_TTL_MS; - let cache: { at: number; data: Map } | undefined; - return async () => { - const now = Date.now(); - if (cache && now - cache.at < ttlMs) { - return cache.data; + let inflight: Promise> | undefined; + let cached: { at: number; data: Map } | undefined; + return () => { + if (inflight) { + return inflight; } - const data = await queryEntitiesCountByKind(knex); - cache = { at: now, data }; - return data; + if (cached && Date.now() - cached.at < ttlMs) { + return Promise.resolve(cached.data); + } + inflight = (async () => { + try { + const data = await queryEntitiesCountByKind(knex); + cached = { at: Date.now(), data }; + return data; + } finally { + inflight = undefined; + } + })(); + return inflight; }; }