diff --git a/.changeset/catalog-metrics-cache.md b/.changeset/catalog-metrics-cache.md new file mode 100644 index 0000000000..bbe427bcea --- /dev/null +++ b/.changeset/catalog-metrics-cache.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Improved the performance of the `catalog_entities_count` metric. + +The legacy Prometheus and OpenTelemetry observable gauges previously each ran their own copy of the per-kind count query against the `search` table on every metrics scrape. On large catalogs this could pile up faster than the queries completed, contending for buffers and stalling the database. + +The two callbacks now share a single query result with a short in-process TTL cache, and the underlying query reads from `final_entities` instead of `search`, avoiding the bitmap heap scans that dominated the previous form. The emitted labels and values are unchanged. diff --git a/plugins/catalog-backend/src/database/metrics.test.ts b/plugins/catalog-backend/src/database/metrics.test.ts new file mode 100644 index 0000000000..74613dcc91 --- /dev/null +++ b/plugins/catalog-backend/src/database/metrics.test.ts @@ -0,0 +1,162 @@ +/* + * Copyright 2026 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; +import { Knex } from 'knex'; +import { randomUUID as uuid } from 'node:crypto'; +import { applyDatabaseMigrations } from './migrations'; +import { DbFinalEntitiesRow, DbRefreshStateRow } from './tables'; +import { createEntitiesCountByKind, queryEntitiesCountByKind } from './metrics'; + +jest.setTimeout(60_000); + +describe('metrics', () => { + const databases = TestDatabases.create(); + + async function createDatabase(databaseId: TestDatabaseId) { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + return knex; + } + + async function insertEntity( + knex: Knex, + options: { entityRef: string; finalEntity: string | null }, + ): Promise { + const entityId = uuid(); + await knex('refresh_state').insert({ + entity_id: entityId, + entity_ref: options.entityRef, + unprocessed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + await knex('final_entities').insert({ + entity_id: entityId, + entity_ref: options.entityRef, + hash: 'h', + final_entity: options.finalEntity ?? undefined, + last_updated_at: '2021-04-01 13:37:00', + }); + } + + describe('queryEntitiesCountByKind', () => { + it.each(databases.eachSupportedId())( + 'counts entities grouped by the kind in entity_ref, %p', + async databaseId => { + const knex = await createDatabase(databaseId); + + await insertEntity(knex, { + entityRef: 'component:default/svc-a', + finalEntity: '{"kind":"Component"}', + }); + await insertEntity(knex, { + entityRef: 'component:default/svc-b', + finalEntity: '{"kind":"Component"}', + }); + await insertEntity(knex, { + entityRef: 'api:default/api-a', + finalEntity: '{"kind":"API"}', + }); + await insertEntity(knex, { + entityRef: 'system:other/sys-a', + finalEntity: '{"kind":"System"}', + }); + // Not yet stitched -- must be excluded from the count + await insertEntity(knex, { + entityRef: 'component:default/pending', + finalEntity: null, + }); + + const result = await queryEntitiesCountByKind(knex); + + expect(Object.fromEntries(result)).toEqual({ + component: 2, + api: 1, + system: 1, + }); + }, + ); + }); + + describe('createEntitiesCountByKind', () => { + it.each(databases.eachSupportedId())( + 'serves cached results within the TTL and refreshes after, %p', + async databaseId => { + const knex = await createDatabase(databaseId); + const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 }); + + await insertEntity(knex, { + entityRef: 'component:default/one', + finalEntity: '{}', + }); + + const first = await getCount(); + expect(Object.fromEntries(first)).toEqual({ component: 1 }); + + // A change made within the TTL window must not be visible yet. + await insertEntity(knex, { + entityRef: 'component:default/two', + finalEntity: '{}', + }); + const cached = await getCount(); + expect(Object.fromEntries(cached)).toEqual({ component: 1 }); + + // After the TTL elapses the next call hits the database again. + await new Promise(resolve => setTimeout(resolve, 80)); + const refreshed = await getCount(); + expect(Object.fromEntries(refreshed)).toEqual({ component: 2 }); + }, + ); + + it.each(databases.eachSupportedId())( + 'coalesces overlapping callers into a single underlying query, %p', + async databaseId => { + const knex = await createDatabase(databaseId); + const getCount = createEntitiesCountByKind(knex, { ttlMs: 50 }); + + await insertEntity(knex, { + entityRef: 'component:default/one', + finalEntity: '{}', + }); + + const finalEntitiesQueries: string[] = []; + knex.on('query', (q: { sql: string }) => { + if ( + /from\s+["`]?final_entities["`]?/i.test(q.sql) && + /^\s*select/i.test(q.sql) + ) { + finalEntitiesQueries.push(q.sql); + } + }); + + // Five concurrent callers should result in one query, not five. + const results = await Promise.all([ + getCount(), + getCount(), + getCount(), + getCount(), + getCount(), + ]); + expect(finalEntitiesQueries).toHaveLength(1); + for (const r of results) { + expect(Object.fromEntries(r)).toEqual({ component: 1 }); + } + }, + ); + }); +}); diff --git a/plugins/catalog-backend/src/database/metrics.ts b/plugins/catalog-backend/src/database/metrics.ts index f6a1464b3a..40a3c62b7a 100644 --- a/plugins/catalog-backend/src/database/metrics.ts +++ b/plugins/catalog-backend/src/database/metrics.ts @@ -16,12 +16,102 @@ import { Knex } from 'knex'; import { createGaugeMetric } from '../util/metrics'; -import { DbRelationsRow, DbLocationsRow, DbSearchRow } from './tables'; +import { DbRelationsRow, DbLocationsRow } from './tables'; import { MetricsService } from '@backstage/backend-plugin-api/alpha'; +const ENTITIES_COUNT_TTL_MS = 30_000; + +/** + * Returns a function that produces a Map of entity kind -> count, with + * a single-flight cache that coalesces overlapping calls. + * + * The OpenTelemetry observable gauge and the legacy Prometheus gauge are + * both registered to emit the same `catalog_entities_count` series, and + * both fire on every metrics scrape. Without coalescing, that means two + * identical heavy queries per scrape per pod, which can pile up against + * the database faster than they complete. + * + * Concurrent callers share the same in-flight promise, so a query is + * never overlapped by another instance of itself. The TTL is the + * minimum age at which the next caller is allowed to start a fresh + * query; if a query is still running when the TTL would have elapsed, + * waiting callers continue to share that one rather than starting a + * duplicate. + * + * @internal exported for testing + */ +export function createEntitiesCountByKind( + knex: Knex, + options?: { ttlMs?: number }, +): () => Promise> { + const ttlMs = options?.ttlMs ?? ENTITIES_COUNT_TTL_MS; + let inflight: Promise> | undefined; + let cached: { at: number; data: Map } | undefined; + return () => { + if (inflight) { + return inflight; + } + if (cached && Date.now() - cached.at < ttlMs) { + return Promise.resolve(cached.data); + } + inflight = (async () => { + try { + const data = await queryEntitiesCountByKind(knex); + cached = { at: Date.now(), data }; + return data; + } finally { + inflight = undefined; + } + })(); + return inflight; + }; +} + +/** + * Reads kind counts straight from `final_entities` (one row per entity) + * rather than from `search` (one row per entity per indexed key, often + * 20-30x larger). The kind is parsed out of `entity_ref`, which is the + * canonical lowercased `kind:namespace/name` form -- producing the same + * lowercased labels the previous search-based query did. + * + * Rows where `final_entity` is null are excluded. They represent entities + * that haven't been stitched yet, or tombstones for in-progress deletions. + * Counting them would over-report by including entities the rest of the + * catalog API treats as not present. + * + * @internal exported for testing + */ +export async function queryEntitiesCountByKind( + knex: Knex, +): Promise> { + const kindExpr = entityRefKindExpression(knex); + + const rows: { kind: string; count: string | number }[] = await knex( + 'final_entities', + ) + .whereNotNull('final_entity') + .select({ kind: kindExpr, count: knex.raw('count(*)') }) + .groupBy(kindExpr); + + return new Map(rows.map(row => [String(row.kind), Number(row.count)])); +} + +function entityRefKindExpression(knex: Knex): Knex.Raw { + const client = knex.client.config.client as string; + if (client.includes('pg')) { + return knex.raw(`split_part(entity_ref, ':', 1)`); + } + if (client.includes('mysql')) { + return knex.raw(`substring_index(entity_ref, ':', 1)`); + } + // sqlite (better-sqlite3, sqlite3) + return knex.raw(`substr(entity_ref, 1, instr(entity_ref, ':') - 1)`); +} + export function initDatabaseMetrics(knex: Knex, metrics: MetricsService) { const seenProm = new Set(); const seen = new Set(); + const getEntitiesCountByKind = createEntitiesCountByKind(knex); return { entities_count_prom: createGaugeMetric({ @@ -29,24 +119,20 @@ export function initDatabaseMetrics(knex: Knex, metrics: MetricsService) { help: 'Total amount of entities in the catalog. DEPRECATED: Please use opentelemetry metrics instead.', labelNames: ['kind'], async collect() { - const results = await knex('search') - .where('key', '=', 'kind') - .whereNotNull('value') - .select({ kind: 'value', count: knex.raw('count(*)') }) - .groupBy('value'); + const results = await getEntitiesCountByKind(); - results.forEach(({ kind, count }) => { + for (const [kind, count] of results) { seenProm.add(kind); - this.set({ kind }, Number(count)); - }); + this.set({ kind }, count); + } // Set all the entities that were not seenProm to 0 and delete them from the seenProm set. - seenProm.forEach(kind => { - if (!results.some(r => r.kind === kind)) { + for (const kind of seenProm) { + if (!results.has(kind)) { this.set({ kind }, 0); seenProm.delete(kind); } - }); + } }, }), registered_locations_prom: createGaugeMetric({ @@ -74,24 +160,20 @@ export function initDatabaseMetrics(knex: Knex, metrics: MetricsService) { description: 'Total amount of entities in the catalog', }) .addCallback(async gauge => { - const results = await knex('search') - .where('key', '=', 'kind') - .whereNotNull('value') - .select({ kind: 'value', count: knex.raw('count(*)') }) - .groupBy('value'); + const results = await getEntitiesCountByKind(); - results.forEach(({ kind, count }) => { + for (const [kind, count] of results) { seen.add(kind); - gauge.observe(Number(count), { kind }); - }); + gauge.observe(count, { kind }); + } // Set all the entities that were not seen to 0 and delete them from the seen set. - seen.forEach(kind => { - if (!results.some(r => r.kind === kind)) { + for (const kind of seen) { + if (!results.has(kind)) { gauge.observe(0, { kind }); seen.delete(kind); } - }); + } }), registered_locations: metrics .createObservableGauge('catalog_registered_locations_count', {