diff --git a/.changeset/plenty-mirrors-bow.md b/.changeset/plenty-mirrors-bow.md new file mode 100644 index 0000000000..d2686b60d3 --- /dev/null +++ b/.changeset/plenty-mirrors-bow.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Add experimental Prometheus metrics instrumentation to the catalog diff --git a/packages/backend/package.json b/packages/backend/package.json index b2877dedf5..8f0e84eac9 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -58,9 +58,11 @@ "example-app": "^0.2.41", "express": "^4.17.1", "express-promise-router": "^4.1.0", + "express-prom-bundle": "^6.3.6", "knex": "^0.95.1", "pg": "^8.3.0", "pg-connection-string": "^2.3.0", + "prom-client": "^13.2.0", "sqlite3": "^5.0.1", "winston": "^3.2.1" }, diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index aecb233581..78c6c4c20b 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -36,6 +36,7 @@ import { } from '@backstage/backend-common'; import { Config } from '@backstage/config'; import healthcheck from './plugins/healthcheck'; +import { metricsInit, metricsHandler } from './metrics'; import auth from './plugins/auth'; import catalog from './plugins/catalog'; import codeCoverage from './plugins/codecoverage'; @@ -72,6 +73,7 @@ function makeCreateEnv(config: Config) { } async function main() { + metricsInit(); const logger = getRootLogger(); logger.info( @@ -124,6 +126,7 @@ async function main() { const service = createServiceBuilder(module) .loadConfig(config) .addRouter('', await healthcheck(healthcheckEnv)) + .addRouter('', metricsHandler()) .addRouter('/api', apiRouter) .addRouter('', await app(appEnv)); diff --git a/packages/backend/src/metrics.ts b/packages/backend/src/metrics.ts new file mode 100644 index 0000000000..8834022cd9 --- /dev/null +++ b/packages/backend/src/metrics.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { useHotCleanup } from '@backstage/backend-common'; +import { RequestHandler, Request } from 'express'; +import promBundle from 'express-prom-bundle'; +import prom from 'prom-client'; +import * as url from 'url'; + +/** + * Experimental Prometheus metrics used to benchmark the performance of the + * software catalog. Use this at your own risk. + */ +const rootRegEx = new RegExp('^/([^/]*)/.*'); +const apiRegEx = new RegExp('^/api/([^/]*)/.*'); + +function normalizePath(req: Request): string { + const path = url.parse(req.originalUrl || req.url).pathname || '/'; + + // Capture /api/ and the plugin name + if (apiRegEx.test(path)) { + return path.replace(apiRegEx, '/api/$1'); + } + + // Only the first path segment at root level + return path.replace(rootRegEx, '/$1'); +} + +export function metricsInit(): void { + prom.collectDefaultMetrics({ prefix: 'backstage_' }); +} + +/** + * Adds a /metrics endpoint, register default runtime metrics and instrument the router. + */ +export function metricsHandler(): RequestHandler { + // We can only initialize the metrics once and have to clean them up between hot reloads + useHotCleanup(module, () => prom.register.clear()); + + return promBundle({ + includeMethod: true, + includePath: true, + // Using includePath alone is problematic, as it will include path labels with high + // cardinality (e.g. path params). Instead we would have to template them. However, this + // is difficult, as every backend plugin might use different routes. Instead we only take + // the first directory of the path, to have at least an idea how each plugin performs: + normalizePath, + promClient: { collectDefaultMetrics: {} }, + }); +} diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index f095f39785..c8b34015e2 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -51,8 +51,10 @@ "glob": "^7.1.6", "knex": "^0.95.1", "lodash": "^4.17.15", + "luxon": "^2.0.2", "morgan": "^1.10.0", "p-limit": "^3.0.2", + "prom-client": "^13.2.0", "qs": "^6.9.4", "uuid": "^8.0.0", "winston": "^3.2.1", diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts index 898ef08f6c..7d4d2ef8db 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts @@ -16,6 +16,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { Hash } from 'crypto'; +import { DateTime } from 'luxon'; import waitForExpect from 'wait-for-expect'; import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase'; import { DefaultCatalogProcessingEngine } from './DefaultCatalogProcessingEngine'; @@ -84,7 +85,7 @@ describe('DefaultCatalogProcessingEngine', () => { }, resultHash: '', state: new Map(), - nextUpdateAt: '', + nextUpdateAt: DateTime.now().toSQL(), lastDiscoveryAt: '', }, ], @@ -147,7 +148,7 @@ describe('DefaultCatalogProcessingEngine', () => { }, resultHash: '', state: new Map(), - nextUpdateAt: '', + nextUpdateAt: DateTime.now().toSQL(), lastDiscoveryAt: '', }, ], @@ -181,7 +182,7 @@ describe('DefaultCatalogProcessingEngine', () => { unprocessedEntity: entity, resultHash: 'the matching hash', state: new Map(), - nextUpdateAt: '', + nextUpdateAt: DateTime.now().toSQL(), lastDiscoveryAt: '', }; diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts index 6468fa5187..41e81b1777 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts @@ -22,8 +22,10 @@ import { import { serializeError } from '@backstage/errors'; import { Hash } from 'crypto'; import stableStringify from 'fast-json-stable-stringify'; +import { DateTime } from 'luxon'; import { Logger } from 'winston'; import { ProcessingDatabase, RefreshStateItem } from './database/types'; +import { createCounterMetric, createSummaryMetric } from './metrics'; import { CatalogProcessingOrchestrator } from './processing/types'; import { Stitcher } from './stitching/Stitcher'; import { startTaskPipeline } from './TaskPipeline'; @@ -84,6 +86,20 @@ class Connection implements EntityProviderConnection { export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { private stopFunc?: () => void; + private readonly metrics = { + processedEntities: createCounterMetric({ + name: 'catalog_processed_entities_count', + help: 'Amount of entities processed', + }), + processingDuration: createSummaryMetric({ + name: 'catalog_processing_duration_seconds', + help: 'Processing duration', + }), + processingQueueDelay: createSummaryMetric({ + name: 'catalog_processing_queue_delay_seconds', + help: 'The amount of delay between being scheduled for processing, and the start of actually being processed', + }), + }; constructor( private readonly logger: Logger, @@ -95,6 +111,10 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { ) {} async start() { + if (this.stopFunc) { + throw new Error('Processing engine is already started'); + } + for (const provider of this.entityProviders) { await provider.connect( new Connection({ @@ -104,10 +124,6 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { ); } - if (this.stopFunc) { - throw new Error('Processing engine is already started'); - } - this.stopFunc = startTaskPipeline({ lowWatermark: 5, highWatermark: 10, @@ -127,7 +143,16 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { } }, processTask: async item => { + let endTimer; try { + this.metrics.processedEntities.inc(1); + this.metrics.processingQueueDelay.observe( + -DateTime.fromSQL(item.nextUpdateAt, { zone: 'UTC' }) + .diffNow() + .as('seconds'), + ); + endTimer = this.metrics.processingDuration.startTimer(); + const { id, state, @@ -213,6 +238,8 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { await this.stitcher.stitch(setOfThingsToStitch); } catch (error) { this.logger.warn('Processing failed with:', error); + } finally { + endTimer?.(); } }, }); diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 203ca3efbf..7326a25a63 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -24,6 +24,7 @@ import type { Logger } from 'winston'; import { Transaction } from '../../database'; import { DeferredEntity } from '../processing/types'; import { RefreshIntervalFunction } from '../refresh'; +import { initDatabaseMetrics } from './metrics'; import { DbRefreshStateReferencesRow, DbRefreshStateRow, @@ -51,7 +52,9 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { logger: Logger; refreshInterval: RefreshIntervalFunction; }, - ) {} + ) { + initDatabaseMetrics(options.database); + } async updateProcessedEntity( txOpaque: Transaction, diff --git a/plugins/catalog-backend/src/next/database/metrics.ts b/plugins/catalog-backend/src/next/database/metrics.ts new file mode 100644 index 0000000000..a41ad512f0 --- /dev/null +++ b/plugins/catalog-backend/src/next/database/metrics.ts @@ -0,0 +1,72 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; +import { DbLocationsRow } from '../../database/types'; +import { createGaugeMetric } from '../metrics'; +import { DbRefreshStateRow, DbRelationsRow } from './tables'; + +export function initDatabaseMetrics(knex: Knex) { + const seen = new Set(); + return { + entities_count: createGaugeMetric({ + name: 'catalog_entities_count', + help: 'Total amount of entities in the catalog', + labelNames: ['kind'], + async collect() { + const result = await knex('refresh_state').select( + 'entity_ref', + ); + const results = result + .map(row => row.entity_ref.split(':')[0]) + .reduce((acc, e) => acc.set(e, (acc.get(e) || 0) + 1), new Map()); + + results.forEach((value, key) => { + seen.add(key); + this.set({ kind: key }, value); + }); + + // Set all the entities that were not seen to 0 and delete them from the seen set. + seen.forEach(key => { + if (!results.has(key)) { + this.set({ kind: key }, 0); + seen.delete(key); + } + }); + }, + }), + registered_locations: createGaugeMetric({ + name: 'catalog_registered_locations_count', + help: 'Total amount of registered locations in the catalog', + async collect() { + const total = await knex('locations').count({ + count: '*', + }); + this.set(Number(total[0].count)); + }, + }), + relations: createGaugeMetric({ + name: 'catalog_relations_count', + help: 'Total amount of relations between entities', + async collect() { + const total = await knex('relations').count({ + count: '*', + }); + this.set(Number(total[0].count)); + }, + }), + }; +} diff --git a/plugins/catalog-backend/src/next/metrics.ts b/plugins/catalog-backend/src/next/metrics.ts new file mode 100644 index 0000000000..4988af82ea --- /dev/null +++ b/plugins/catalog-backend/src/next/metrics.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Counter, + CounterConfiguration, + Gauge, + GaugeConfiguration, + Histogram, + HistogramConfiguration, + register, + Summary, + SummaryConfiguration, +} from 'prom-client'; + +export function createCounterMetric( + config: CounterConfiguration, +): Counter { + const existing = register.getSingleMetric(config.name) as Counter; + return existing || new Counter(config); +} + +export function createGaugeMetric( + config: GaugeConfiguration, +): Gauge { + const existing = register.getSingleMetric(config.name) as Gauge; + return existing || new Gauge(config); +} + +export function createSummaryMetric( + config: SummaryConfiguration, +): Summary { + const existing = register.getSingleMetric(config.name) as Summary; + return existing || new Summary(config); +} + +export function createHistogramMetric( + config: HistogramConfiguration, +): Histogram { + const existing = register.getSingleMetric(config.name) as Histogram; + return existing || new Histogram(config); +} diff --git a/yarn.lock b/yarn.lock index 2fe37fc657..f86c3fd0b4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9446,6 +9446,11 @@ bindings@^1.5.0: dependencies: file-uri-to-path "1.0.0" +bintrees@1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz#0e655c9b9c2435eaab68bf4027226d2b55a34524" + integrity sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ= + bl@^4.0.3, bl@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" @@ -13475,6 +13480,14 @@ expect@^26.6.2: jest-message-util "^26.6.2" jest-regex-util "^26.0.0" +express-prom-bundle@^6.3.6: + version "6.3.6" + resolved "https://registry.npmjs.org/express-prom-bundle/-/express-prom-bundle-6.3.6.tgz#c8da1c1024edfcc54953c365991aca57ffd0cfda" + integrity sha512-IRsTRCEKCVCHEriQlZ1FuutjEFc89KASsveXh+1HcGEnuZKiAC4LugxrsGEIdySqYvqOYSr2SWHJ6L8/BK2SHA== + dependencies: + on-finished "^2.3.0" + url-value-parser "^2.0.0" + express-promise-router@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/express-promise-router/-/express-promise-router-4.1.0.tgz#79160e145c27610ba411bceb0552a36f11dbab4f" @@ -20451,7 +20464,7 @@ oidc-token-hash@^5.0.1: resolved "https://registry.npmjs.org/oidc-token-hash/-/oidc-token-hash-5.0.1.tgz#ae6beec3ec20f0fd885e5400d175191d6e2f10c6" integrity sha512-EvoOtz6FIEBzE+9q253HsLCVRiK/0doEJ2HCvvqMQb3dHZrP3WlJKYtJ55CRTw4jmYomzH4wkPuCj/I3ZvpKxQ== -on-finished@~2.3.0: +on-finished@^2.3.0, on-finished@~2.3.0: version "2.3.0" resolved "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" integrity sha1-IPEzZIGwg811M3mSoWlxqi2QaUc= @@ -22089,6 +22102,13 @@ progress@^2.0.0: resolved "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== +prom-client@^13.2.0: + version "13.2.0" + resolved "https://registry.npmjs.org/prom-client/-/prom-client-13.2.0.tgz#99d13357912dd400f8911b77df19f7b328a93e92" + integrity sha512-wGr5mlNNdRNzEhRYXgboUU2LxHWIojxscJKmtG3R8f4/KiWqyYgXTLHs0+Ted7tG3zFT7pgHJbtomzZ1L0ARaQ== + dependencies: + tdigest "^0.1.1" + promise-inflight@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3" @@ -25669,6 +25689,13 @@ tarn@^3.0.1: resolved "https://registry.npmjs.org/tarn/-/tarn-3.0.1.tgz#ebac2c6dbc6977d34d4526e0a7814200386a8aec" integrity sha512-6usSlV9KyHsspvwu2duKH+FMUhqJnAh6J5J/4MITl8s94iSUQTLkJggdiewKv4RyARQccnigV48Z+khiuVZDJw== +tdigest@^0.1.1: + version "0.1.1" + resolved "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz#2e3cb2c39ea449e55d1e6cd91117accca4588021" + integrity sha1-Ljyyw56kSeVdHmzZEReszKRYgCE= + dependencies: + bintrees "1.0.1" + teeny-request@^3.11.3: version "3.11.3" resolved "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz#335c629f7645e5d6599362df2f3230c4cbc23a55" @@ -26847,6 +26874,11 @@ url-parse@^1.4.3, url-parse@^1.5.1: querystringify "^2.1.1" requires-port "^1.0.0" +url-value-parser@^2.0.0: + version "2.0.3" + resolved "https://registry.npmjs.org/url-value-parser/-/url-value-parser-2.0.3.tgz#cd4b8d6754e458d65e8125260c09718d926e6e21" + integrity sha512-FjIX+Q9lYmDM9uYIGdMYfQW0uLbWVwN2NrL2ayAI7BTOvEwzH+VoDdNquwB9h4dFAx+u6mb0ONLa3sHD5DvyvA== + url@0.10.3: version "0.10.3" resolved "https://registry.npmjs.org/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64"