From 8ff2e84ddbe12be797b2af223173edcc65a73599 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 22 Sep 2023 16:10:59 +0200 Subject: [PATCH 1/4] feat: move over these metrics to open telemetry too Signed-off-by: blam --- .../catalog-backend/src/database/metrics.ts | 62 ++++++++++++++++--- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/plugins/catalog-backend/src/database/metrics.ts b/plugins/catalog-backend/src/database/metrics.ts index b6c4e248dc..58ad91f93b 100644 --- a/plugins/catalog-backend/src/database/metrics.ts +++ b/plugins/catalog-backend/src/database/metrics.ts @@ -17,11 +17,14 @@ import { Knex } from 'knex'; import { createGaugeMetric } from '../util/metrics'; import { DbRefreshStateRow, DbRelationsRow, DbLocationsRow } from './tables'; +import { metrics } from '@opentelemetry/api'; export function initDatabaseMetrics(knex: Knex) { + const seenProm = new Set(); const seen = new Set(); + const meter = metrics.getMeter('default'); return { - entities_count: createGaugeMetric({ + entities_count_prom: createGaugeMetric({ name: 'catalog_entities_count', help: 'Total amount of entities in the catalog', labelNames: ['kind'], @@ -34,20 +37,20 @@ export function initDatabaseMetrics(knex: Knex) { .reduce((acc, e) => acc.set(e, (acc.get(e) || 0) + 1), new Map()); results.forEach((value, key) => { - seen.add(key); + seenProm.add(key); this.set({ kind: key }, value); }); - // Set all the entities that were not seen to 0 and delete them from the seen set. - seen.forEach(key => { + // Set all the entities that were not seenProm to 0 and delete them from the seenProm set. + seenProm.forEach(key => { if (!results.has(key)) { this.set({ kind: key }, 0); - seen.delete(key); + seenProm.delete(key); } }); }, }), - registered_locations: createGaugeMetric({ + registered_locations_prom: createGaugeMetric({ name: 'catalog_registered_locations_count', help: 'Total amount of registered locations in the catalog', async collect() { @@ -57,7 +60,7 @@ export function initDatabaseMetrics(knex: Knex) { this.set(Number(total[0].count)); }, }), - relations: createGaugeMetric({ + relations_prom: createGaugeMetric({ name: 'catalog_relations_count', help: 'Total amount of relations between entities', async collect() { @@ -67,5 +70,50 @@ export function initDatabaseMetrics(knex: Knex) { this.set(Number(total[0].count)); }, }), + entities_count: meter + .createObservableGauge('catalog_entities_count', { + description: 'Total amount of entities in the catalog', + }) + .addCallback(async gauge => { + const result = await knex('refresh_state').select( + 'entity_ref', + ); + const results = result + .map(row => row.entity_ref.split(':')[0]) + .reduce((acc, e) => acc.set(e, (acc.get(e) || 0) + 1), new Map()); + + results.forEach((value, key) => { + seen.add(key); + gauge.observe(value, { kind: key }); + }); + + // Set all the entities that were not seen to 0 and delete them from the seen set. + seen.forEach(key => { + if (!results.has(key)) { + gauge.observe(0, { kind: key }); + seen.delete(key); + } + }); + }), + registered_locations: meter + .createObservableGauge('catalog_registered_locations_count', { + description: 'Total amount of registered locations in the catalog', + }) + .addCallback(async gauge => { + const total = await knex('locations').count({ + count: '*', + }); + gauge.observe(Number(total[0].count)); + }), + relations: meter + .createObservableGauge('catalog_relations_count', { + description: 'Total amount of relations between entities', + }) + .addCallback(async gauge => { + const total = await knex('relations').count({ + count: '*', + }); + gauge.observe(Number(total[0].count)); + }), }; } From 8770d122564a700913b7a8738f9a185a0d48b6b8 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 22 Sep 2023 16:20:13 +0200 Subject: [PATCH 2/4] chore: added deprecated notice Signed-off-by: blam --- plugins/catalog-backend/src/database/metrics.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/src/database/metrics.ts b/plugins/catalog-backend/src/database/metrics.ts index 58ad91f93b..55bfa66b86 100644 --- a/plugins/catalog-backend/src/database/metrics.ts +++ b/plugins/catalog-backend/src/database/metrics.ts @@ -26,7 +26,7 @@ export function initDatabaseMetrics(knex: Knex) { return { entities_count_prom: createGaugeMetric({ name: 'catalog_entities_count', - help: 'Total amount of entities in the catalog', + help: 'Total amount of entities in the catalog. DEPRECATED: Please use opentelemetry metrics instead.', labelNames: ['kind'], async collect() { const result = await knex('refresh_state').select( @@ -52,7 +52,7 @@ export function initDatabaseMetrics(knex: Knex) { }), registered_locations_prom: createGaugeMetric({ name: 'catalog_registered_locations_count', - help: 'Total amount of registered locations in the catalog', + help: 'Total amount of registered locations in the catalog. DEPRECATED: Please use opentelemetry metrics instead.', async collect() { const total = await knex('locations').count({ count: '*', @@ -62,7 +62,7 @@ export function initDatabaseMetrics(knex: Knex) { }), relations_prom: createGaugeMetric({ name: 'catalog_relations_count', - help: 'Total amount of relations between entities', + help: 'Total amount of relations between entities. DEPRECATED: Please use opentelemetry metrics instead.', async collect() { const total = await knex('relations').count({ count: '*', From 78af9433c8d1a31d4e37e80c96c884564e6261a4 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 22 Sep 2023 16:21:07 +0200 Subject: [PATCH 3/4] feat: added changeset Signed-off-by: blam --- .changeset/spotty-badgers-wash.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/spotty-badgers-wash.md diff --git a/.changeset/spotty-badgers-wash.md b/.changeset/spotty-badgers-wash.md new file mode 100644 index 0000000000..1ce9c66aee --- /dev/null +++ b/.changeset/spotty-badgers-wash.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Instrumenting some missing metrics with `OpenTelemetry` From 54299c496efc61acf78e2bedf4469219e300c572 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 22 Sep 2023 16:25:09 +0200 Subject: [PATCH 4/4] chore: fixing parsing Signed-off-by: blam --- plugins/catalog-backend/src/database/metrics.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/database/metrics.ts b/plugins/catalog-backend/src/database/metrics.ts index 55bfa66b86..8eb307ea46 100644 --- a/plugins/catalog-backend/src/database/metrics.ts +++ b/plugins/catalog-backend/src/database/metrics.ts @@ -18,6 +18,7 @@ import { Knex } from 'knex'; import { createGaugeMetric } from '../util/metrics'; import { DbRefreshStateRow, DbRelationsRow, DbLocationsRow } from './tables'; import { metrics } from '@opentelemetry/api'; +import { parseEntityRef } from '@backstage/catalog-model'; export function initDatabaseMetrics(knex: Knex) { const seenProm = new Set(); @@ -79,7 +80,7 @@ export function initDatabaseMetrics(knex: Knex) { 'entity_ref', ); const results = result - .map(row => row.entity_ref.split(':')[0]) + .map(row => parseEntityRef(row.entity_ref).kind) .reduce((acc, e) => acc.set(e, (acc.get(e) || 0) + 1), new Map()); results.forEach((value, key) => {