From 2924ffe09bf5dab6fc083ec247e7ef8f0e6bf053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sun, 24 Nov 2024 15:14:17 +0100 Subject: [PATCH 1/2] Compute some metrics using search table facet aggregations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/young-carrots-collect.md | 5 ++ .../catalog-backend/src/database/metrics.ts | 53 +++++++++---------- 2 files changed, 30 insertions(+), 28 deletions(-) create mode 100644 .changeset/young-carrots-collect.md diff --git a/.changeset/young-carrots-collect.md b/.changeset/young-carrots-collect.md new file mode 100644 index 0000000000..1350400d39 --- /dev/null +++ b/.changeset/young-carrots-collect.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Compute some metrics using search table facet aggregations instead of reading the full refresh state diff --git a/plugins/catalog-backend/src/database/metrics.ts b/plugins/catalog-backend/src/database/metrics.ts index 8eb307ea46..e7032d8263 100644 --- a/plugins/catalog-backend/src/database/metrics.ts +++ b/plugins/catalog-backend/src/database/metrics.ts @@ -16,9 +16,8 @@ import { Knex } from 'knex'; import { createGaugeMetric } from '../util/metrics'; -import { DbRefreshStateRow, DbRelationsRow, DbLocationsRow } from './tables'; +import { DbRelationsRow, DbLocationsRow, DbSearchRow } from './tables'; import { metrics } from '@opentelemetry/api'; -import { parseEntityRef } from '@backstage/catalog-model'; export function initDatabaseMetrics(knex: Knex) { const seenProm = new Set(); @@ -30,23 +29,22 @@ export function initDatabaseMetrics(knex: Knex) { help: 'Total amount of entities in the catalog. DEPRECATED: Please use opentelemetry metrics instead.', labelNames: ['kind'], async collect() { - const result = await knex('refresh_state').select( - 'entity_ref', - ); - const results = result - .map(row => row.entity_ref.split(':')[0]) - .reduce((acc, e) => acc.set(e, (acc.get(e) || 0) + 1), new Map()); + const results = await knex('search') + .where('key', '=', 'kind') + .whereNotNull('original_value') + .select({ kind: 'original_value', count: knex.raw('count(*)') }) + .groupBy('original_value'); - results.forEach((value, key) => { - seenProm.add(key); - this.set({ kind: key }, value); + results.forEach(({ kind, count }) => { + seenProm.add(kind); + this.set({ kind }, Number(count)); }); // Set all the entities that were not seenProm to 0 and delete them from the seenProm set. - seenProm.forEach(key => { - if (!results.has(key)) { - this.set({ kind: key }, 0); - seenProm.delete(key); + seenProm.forEach(kind => { + if (!results.some(r => r.kind === kind)) { + this.set({ kind }, 0); + seenProm.delete(kind); } }); }, @@ -76,23 +74,22 @@ export function initDatabaseMetrics(knex: Knex) { description: 'Total amount of entities in the catalog', }) .addCallback(async gauge => { - const result = await knex('refresh_state').select( - 'entity_ref', - ); - const results = result - .map(row => parseEntityRef(row.entity_ref).kind) - .reduce((acc, e) => acc.set(e, (acc.get(e) || 0) + 1), new Map()); + const results = await knex('search') + .where('key', '=', 'kind') + .whereNotNull('original_value') + .select({ kind: 'original_value', count: knex.raw('count(*)') }) + .groupBy('original_value'); - results.forEach((value, key) => { - seen.add(key); - gauge.observe(value, { kind: key }); + results.forEach(({ kind, count }) => { + seen.add(kind); + gauge.observe(Number(count), { kind }); }); // Set all the entities that were not seen to 0 and delete them from the seen set. - seen.forEach(key => { - if (!results.has(key)) { - gauge.observe(0, { kind: key }); - seen.delete(key); + seen.forEach(kind => { + if (!results.some(r => r.kind === kind)) { + gauge.observe(0, { kind }); + seen.delete(kind); } }); }), From 0e04b2558252a0d3060bc171f646692f12c7d0f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 25 Nov 2024 13:57:34 +0100 Subject: [PATCH 2/2] do lowercase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- plugins/catalog-backend/src/database/metrics.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend/src/database/metrics.ts b/plugins/catalog-backend/src/database/metrics.ts index e7032d8263..d8cd87558f 100644 --- a/plugins/catalog-backend/src/database/metrics.ts +++ b/plugins/catalog-backend/src/database/metrics.ts @@ -31,9 +31,9 @@ export function initDatabaseMetrics(knex: Knex) { async collect() { const results = await knex('search') .where('key', '=', 'kind') - .whereNotNull('original_value') - .select({ kind: 'original_value', count: knex.raw('count(*)') }) - .groupBy('original_value'); + .whereNotNull('value') + .select({ kind: 'value', count: knex.raw('count(*)') }) + .groupBy('value'); results.forEach(({ kind, count }) => { seenProm.add(kind); @@ -76,9 +76,9 @@ export function initDatabaseMetrics(knex: Knex) { .addCallback(async gauge => { const results = await knex('search') .where('key', '=', 'kind') - .whereNotNull('original_value') - .select({ kind: 'original_value', count: knex.raw('count(*)') }) - .groupBy('original_value'); + .whereNotNull('value') + .select({ kind: 'value', count: knex.raw('count(*)') }) + .groupBy('value'); results.forEach(({ kind, count }) => { seen.add(kind);