diff --git a/.changeset/healthy-waves-compare.md b/.changeset/healthy-waves-compare.md new file mode 100644 index 0000000000..cdd19d1450 --- /dev/null +++ b/.changeset/healthy-waves-compare.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Deprecated Prometheus metrics in favour of OpenTelemtry metrics. diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index fb754161c5..2bae86b38a 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -47,6 +47,7 @@ "@backstage/plugin-scaffolder-common": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@backstage/types": "workspace:^", + "@opentelemetry/api": "^1.3.0", "@types/express": "^4.17.6", "codeowners-utils": "^1.0.2", "core-js": "^3.6.5", diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index bef2878099..c8f159b195 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -23,6 +23,7 @@ import { assertError, serializeError, stringifyError } from '@backstage/errors'; import { Hash } from 'crypto'; import stableStringify from 'fast-json-stable-stringify'; import { Logger } from 'winston'; +import { metrics } from '@opentelemetry/api'; import { ProcessingDatabase, RefreshStateItem } from '../database/types'; import { createCounterMetric, createSummaryMetric } from '../util/metrics'; import { @@ -257,62 +258,123 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { // Helps wrap the timing and logging behaviors function progressTracker() { - const stitchedEntities = createCounterMetric({ + // prom-client metrics are deprecated in favour of OpenTelemetry metrics. + const promStitchedEntities = createCounterMetric({ name: 'catalog_stitched_entities_count', - help: 'Amount of entities stitched', + help: 'Amount of entities stitched. DEPRECATED, use OpenTelemetry metrics instead', }); - const processedEntities = createCounterMetric({ + const promProcessedEntities = createCounterMetric({ name: 'catalog_processed_entities_count', - help: 'Amount of entities processed', + help: 'Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead', labelNames: ['result'], }); - const processingDuration = createSummaryMetric({ + const promProcessingDuration = createSummaryMetric({ name: 'catalog_processing_duration_seconds', - help: 'Time spent executing the full processing flow', + help: 'Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead', labelNames: ['result'], }); - const processorsDuration = createSummaryMetric({ + const promProcessorsDuration = createSummaryMetric({ name: 'catalog_processors_duration_seconds', - help: 'Time spent executing catalog processors', + help: 'Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead', labelNames: ['result'], }); - const processingQueueDelay = createSummaryMetric({ + const promProcessingQueueDelay = createSummaryMetric({ name: 'catalog_processing_queue_delay_seconds', - help: 'The amount of delay between being scheduled for processing, and the start of actually being processed', + help: 'The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead', }); + const meter = metrics.getMeter('default'); + const stitchedEntities = meter.createCounter( + 'catalog.stitched.entities.count', + { + description: 'Amount of entities stitched', + }, + ); + + const processedEntities = meter.createCounter( + 'catalog.processed.entities.count', + { description: 'Amount of entities processed' }, + ); + + const processingDuration = meter.createHistogram( + 'catalog.processing.duration', + { + description: 'Time spent executing the full processing flow', + unit: 'seconds', + }, + ); + + const processorsDuration = meter.createHistogram( + 'catalog.processors.duration', + { + description: 'Time spent executing catalog processors', + unit: 'seconds', + }, + ); + + const processingQueueDelay = meter.createHistogram( + 'catalog.processing.queue.delay', + { + description: + 'The amount of delay between being scheduled for processing, and the start of actually being processed', + unit: 'seconds', + }, + ); + function processStart(item: RefreshStateItem, logger: Logger) { + const startTime = process.hrtime(); + const endOverallTimer = promProcessingDuration.startTimer(); + const endProcessorsTimer = promProcessorsDuration.startTimer(); + logger.debug(`Processing ${item.entityRef}`); if (item.nextUpdateAt) { - processingQueueDelay.observe(-item.nextUpdateAt.diffNow().as('seconds')); + const seconds = -item.nextUpdateAt.diffNow().as('seconds'); + promProcessingQueueDelay.observe(seconds); + processingQueueDelay.record(seconds); } - const endOverallTimer = processingDuration.startTimer(); - const endProcessorsTimer = processorsDuration.startTimer(); + function endTime() { + const delta = process.hrtime(startTime); + return delta[0] + delta[1] / 1e9; + } function markProcessorsCompleted(result: EntityProcessingResult) { endProcessorsTimer({ result: result.ok ? 'ok' : 'failed' }); + processorsDuration.record(endTime(), { + result: result.ok ? 'ok' : 'failed', + }); } function markSuccessfulWithNoChanges() { endOverallTimer({ result: 'unchanged' }); - processedEntities.inc({ result: 'unchanged' }, 1); + promProcessedEntities.inc({ result: 'unchanged' }, 1); + + processingDuration.record(endTime(), { result: 'unchanged' }); + processedEntities.add(1, { result: 'unchanged' }); } function markSuccessfulWithErrors() { endOverallTimer({ result: 'errors' }); - processedEntities.inc({ result: 'errors' }, 1); + promProcessedEntities.inc({ result: 'errors' }, 1); + + processingDuration.record(endTime(), { result: 'errors' }); + processedEntities.add(1, { result: 'errors' }); } function markSuccessfulWithChanges(stitchedCount: number) { endOverallTimer({ result: 'changed' }); - stitchedEntities.inc(stitchedCount); - processedEntities.inc({ result: 'changed' }, 1); + promStitchedEntities.inc(stitchedCount); + promProcessedEntities.inc({ result: 'changed' }, 1); + + processingDuration.record(endTime(), { result: 'changed' }); + stitchedEntities.add(stitchedCount); + processedEntities.add(1, { result: 'changed' }); } function markFailed(error: Error) { - processedEntities.inc({ result: 'failed' }, 1); + promProcessedEntities.inc({ result: 'failed' }, 1); + processedEntities.add(1, { result: 'failed' }); logger.warn(`Processing of ${item.entityRef} failed`, error); } diff --git a/yarn.lock b/yarn.lock index 2cda2a9e27..e35d8d3abc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5245,6 +5245,7 @@ __metadata: "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@backstage/types": "workspace:^" + "@opentelemetry/api": ^1.3.0 "@types/core-js": ^2.5.4 "@types/express": ^4.17.6 "@types/git-url-parse": ^9.0.0 @@ -12458,10 +12459,10 @@ __metadata: languageName: node linkType: hard -"@opentelemetry/api@npm:^1.0.1": - version: 1.0.4 - resolution: "@opentelemetry/api@npm:1.0.4" - checksum: 793e9b5c21666b647a60c58c46c3e00ad1dac38505102b026ad0ef617571d637aca54a18533a73c1e288c95b5ac77e2db17f96467f11833ac1165338e1184260 +"@opentelemetry/api@npm:^1.0.1, @opentelemetry/api@npm:^1.3.0": + version: 1.3.0 + resolution: "@opentelemetry/api@npm:1.3.0" + checksum: 33d284b67b6fab20ff72961d289c6487d3cb27caf7489f0231d7030551f82871e081e744b0390751d8aef3bf1614bd79f854788901a354e15274f552581fb374 languageName: node linkType: hard