From 342f41a324c5d508b0d30a50d81aacab4391e3d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 11 Feb 2026 13:50:05 +0100 Subject: [PATCH] address review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/early-cases-laugh.md | 28 ++++- plugins/catalog-backend/config.d.ts | 43 +++++++ .../providers/DefaultLocationStore.test.ts | 6 +- .../src/providers/DefaultLocationStore.ts | 36 +++++- .../GenericScmEventRefreshProvider.test.ts | 6 +- .../GenericScmEventRefreshProvider.ts | 15 ++- .../src/service/CatalogBuilder.ts | 9 +- .../util/readScmEventHandlingConfig.test.ts | 110 ++++++++++++++++++ .../src/util/readScmEventHandlingConfig.ts | 56 +++++++++ plugins/catalog-node/report-alpha.api.md | 2 +- .../scmEvents/catalogScmEventsServiceRef.ts | 8 +- 11 files changed, 302 insertions(+), 17 deletions(-) create mode 100644 plugins/catalog-backend/src/util/readScmEventHandlingConfig.test.ts create mode 100644 plugins/catalog-backend/src/util/readScmEventHandlingConfig.ts diff --git a/.changeset/early-cases-laugh.md b/.changeset/early-cases-laugh.md index ba03c156f3..ac74ecb8ba 100644 --- a/.changeset/early-cases-laugh.md +++ b/.changeset/early-cases-laugh.md @@ -2,4 +2,30 @@ '@backstage/plugin-catalog-backend': minor --- -Implemented `catalogScmEventsServiceRef` event handling in the builtin entity providers. +Implemented handling of events from the newly introduced alpha +`catalogScmEventsServiceRef` service, in the builtin entity providers. This +allows entities to get refreshed, and locations updated or removed, as a +response to incoming events. In its first iteration, only the GitHub module +implements such event handling however. + +This is not yet enabled by default, but this fact may change in a future +release. To try it out, ensure that you have the latest catalog GitHub module +installed, and set the following in your app-config: + +```yaml +catalog: + scmEvents: true +``` + +Or if you want to pick and choose from the various features: + +```yaml +catalog: + scmEvents: + # refresh (reprocess) upon events? + refresh: true + # automatically unregister locations based on events? (files deleted, repos archived, etc) + unregister: true + # automatically move locations based on events? (repo transferred, file renamed, etc) + move: true +``` diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 95d28d1eae..48dae2d747 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -257,5 +257,48 @@ export interface Config { priority?: number; }; }; + + /** + * Settings that control what to do when receiving messages from the SCM + * events service. + * + * @defaultValue false + * @remarks + * + * This is primarily meant to affect builtin providers in the catalog + * backend such as the location handler, but other providers and processors + * may also read this configuration. + * + * If set to false, disable all handling of SCM events. + * + * If set to true, enable all default handling of SCM events. Note that the + * set of default handling can change over time. + * + * You can also configure individual handlers one by one. + */ + scmEvents?: + | boolean + | { + /** + * Trigger refreshes (reprocessing) of entities that are affected by an + * SCM event. This may include source control file content changes, + * repository status changes, etc. + * + * @defaultValue true + */ + refresh?: boolean; + /** + * Unregister entities that are deleted as a result of an SCM event. + * + * @defaultValue true + */ + unregister?: boolean; + /** + * Move entities that are moved as a result of an SCM event. + * + * @defaultValue true + */ + move?: boolean; + }; }; } diff --git a/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts index a2ffdee053..e1ee5568ae 100644 --- a/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts +++ b/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts @@ -52,7 +52,11 @@ describe('DefaultLocationStore', () => { const knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); const connection = { applyMutation: jest.fn(), refresh: jest.fn() }; - const store = new DefaultLocationStore(knex, mockScmEvents); + const store = new DefaultLocationStore(knex, mockScmEvents, { + refresh: true, + unregister: true, + move: true, + }); await store.connect(connection); return { store, connection, knex }; } diff --git a/plugins/catalog-backend/src/providers/DefaultLocationStore.ts b/plugins/catalog-backend/src/providers/DefaultLocationStore.ts index 6a8967f760..7faf5da58e 100644 --- a/plugins/catalog-backend/src/providers/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/providers/DefaultLocationStore.ts @@ -42,15 +42,22 @@ import { } from '@backstage/plugin-catalog-node/alpha'; import { chunk, uniqBy } from 'lodash'; import parseGitUrl, { type GitUrl } from 'git-url-parse'; +import { ScmEventHandlingConfig } from '../util/readScmEventHandlingConfig'; export class DefaultLocationStore implements LocationStore, EntityProvider { private _connection: EntityProviderConnection | undefined; private readonly db: Knex; private readonly scmEvents: CatalogScmEventsService; + private readonly scmEventHandlingConfig: ScmEventHandlingConfig; - constructor(db: Knex, scmEvents: CatalogScmEventsService) { + constructor( + db: Knex, + scmEvents: CatalogScmEventsService, + scmEventHandlingConfig: ScmEventHandlingConfig, + ) { this.db = db; this.scmEvents = scmEvents; + this.scmEventHandlingConfig = scmEventHandlingConfig; } getProviderName(): string { @@ -194,7 +201,12 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { entities, }); - this.scmEvents.subscribe({ onEvents: this.#onScmEvents.bind(this) }); + if ( + this.scmEventHandlingConfig.unregister || + this.scmEventHandlingConfig.move + ) { + this.scmEvents.subscribe({ onEvents: this.#onScmEvents.bind(this) }); + } } private async locations(dbOrTx: Knex.Transaction | Knex = this.db) { @@ -221,16 +233,28 @@ export class DefaultLocationStore implements LocationStore, EntityProvider { const locationPrefixesToMove = new Map(); for (const event of events) { - if (event.type === 'location.deleted') { + if ( + event.type === 'location.deleted' && + this.scmEventHandlingConfig.unregister + ) { exactLocationsToDelete.add(event.url); - } else if (event.type === 'location.moved') { + } else if ( + event.type === 'location.moved' && + this.scmEventHandlingConfig.move + ) { // Since Location entities are named after their target URL, these // unfortunately have to be translated into deletion and creation exactLocationsToDelete.add(event.fromUrl); exactLocationsToCreate.add(event.toUrl); - } else if (event.type === 'repository.deleted') { + } else if ( + event.type === 'repository.deleted' && + this.scmEventHandlingConfig.unregister + ) { locationPrefixesToDelete.add(event.url); - } else if (event.type === 'repository.moved') { + } else if ( + event.type === 'repository.moved' && + this.scmEventHandlingConfig.move + ) { // These also have to be handled with deletions and creations locationPrefixesToMove.set(event.fromUrl, event.toUrl); } diff --git a/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.test.ts b/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.test.ts index 7db594b8fe..8c74f350bb 100644 --- a/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.test.ts +++ b/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.test.ts @@ -49,7 +49,11 @@ describe('GenericScmEventRefreshProvider', () => { publish: jest.fn(), }; - const store = new GenericScmEventRefreshProvider(knex, scmEvents); + const store = new GenericScmEventRefreshProvider(knex, scmEvents, { + refresh: true, + unregister: true, + move: true, + }); const connection = { applyMutation: jest.fn(), diff --git a/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.ts b/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.ts index 8f21bbf9de..9d62ef1bdf 100644 --- a/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.ts +++ b/plugins/catalog-backend/src/providers/GenericScmEventRefreshProvider.ts @@ -33,21 +33,28 @@ import { DbRefreshStateRow, DbSearchRow, } from '../database/tables'; +import { ScmEventHandlingConfig } from '../util/readScmEventHandlingConfig'; /** * Deals in a generic fashion with SCM events, refreshing entities as needed. * - * It's implemented in the form of an entity provider even though itt actually + * It's implemented in the form of an entity provider even though its actually * does not behave like one, mostly in order to consistently start treating * events on connect time when the engine is known to be ready. */ export class GenericScmEventRefreshProvider implements EntityProvider { readonly #knex: Knex; readonly #scmEvents: CatalogScmEventsService; + readonly #scmEventHandlingConfig: ScmEventHandlingConfig; - constructor(knex: Knex, scmEvents: CatalogScmEventsService) { + constructor( + knex: Knex, + scmEvents: CatalogScmEventsService, + scmEventHandlingConfig: ScmEventHandlingConfig, + ) { this.#knex = knex; this.#scmEvents = scmEvents; + this.#scmEventHandlingConfig = scmEventHandlingConfig; } getProviderName(): string { @@ -55,7 +62,9 @@ export class GenericScmEventRefreshProvider implements EntityProvider { } async connect(_connection: EntityProviderConnection): Promise { - this.#scmEvents.subscribe({ onEvents: this.#onScmEvents.bind(this) }); + if (this.#scmEventHandlingConfig.refresh) { + this.#scmEvents.subscribe({ onEvents: this.#onScmEvents.bind(this) }); + } } async #onScmEvents(events: CatalogScmEvent[]): Promise { diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index db9ef8a867..e3d0c26b0f 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -116,6 +116,7 @@ import { } from '@backstage/plugin-catalog-node/alpha'; import { filterAndSortProcessors, filterProviders } from './util'; import { GenericScmEventRefreshProvider } from '../providers/GenericScmEventRefreshProvider'; +import { readScmEventHandlingConfig } from '../util/readScmEventHandlingConfig'; export type CatalogEnvironment = { logger: LoggerService; @@ -532,11 +533,17 @@ export class CatalogBuilder { }); } - const locationStore = new DefaultLocationStore(dbClient, catalogScmEvents); + const scmEventHandlingConfig = readScmEventHandlingConfig(config); + const locationStore = new DefaultLocationStore( + dbClient, + catalogScmEvents, + scmEventHandlingConfig, + ); const configLocationProvider = new ConfigLocationEntityProvider(config); const scmEvents = new GenericScmEventRefreshProvider( dbClient, catalogScmEvents, + scmEventHandlingConfig, ); const entityProviderEntries = lodash.uniqBy( diff --git a/plugins/catalog-backend/src/util/readScmEventHandlingConfig.test.ts b/plugins/catalog-backend/src/util/readScmEventHandlingConfig.test.ts new file mode 100644 index 0000000000..a99bf2a47c --- /dev/null +++ b/plugins/catalog-backend/src/util/readScmEventHandlingConfig.test.ts @@ -0,0 +1,110 @@ +/* + * Copyright 2026 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { readScmEventHandlingConfig } from './readScmEventHandlingConfig'; + +describe('readScmEventHandlingConfig', () => { + it('returns defaults when catalog.scmEvents is not set', () => { + const config = new ConfigReader({}); + + expect(readScmEventHandlingConfig(config)).toEqual({ + refresh: true, + unregister: true, + move: true, + }); + }); + + it('returns defaults when catalog.scmEvents is true', () => { + const config = new ConfigReader({ + catalog: { + scmEvents: true, + }, + }); + + expect(readScmEventHandlingConfig(config)).toEqual({ + refresh: true, + unregister: true, + move: true, + }); + }); + + it('returns all disabled when catalog.scmEvents is false', () => { + const config = new ConfigReader({ + catalog: { + scmEvents: false, + }, + }); + + expect(readScmEventHandlingConfig(config)).toEqual({ + refresh: false, + unregister: false, + move: false, + }); + }); + + it('merges partial config with defaults', () => { + const config = new ConfigReader({ + catalog: { + scmEvents: { + refresh: false, + }, + }, + }); + + expect(readScmEventHandlingConfig(config)).toEqual({ + refresh: false, + unregister: true, + move: true, + }); + }); + + it('allows disabling individual options', () => { + const config = new ConfigReader({ + catalog: { + scmEvents: { + refresh: true, + unregister: false, + move: false, + }, + }, + }); + + expect(readScmEventHandlingConfig(config)).toEqual({ + refresh: true, + unregister: false, + move: false, + }); + }); + + it('allows enabling all options explicitly', () => { + const config = new ConfigReader({ + catalog: { + scmEvents: { + refresh: true, + unregister: true, + move: true, + }, + }, + }); + + expect(readScmEventHandlingConfig(config)).toEqual({ + refresh: true, + unregister: true, + move: true, + }); + }); +}); diff --git a/plugins/catalog-backend/src/util/readScmEventHandlingConfig.ts b/plugins/catalog-backend/src/util/readScmEventHandlingConfig.ts new file mode 100644 index 0000000000..b2e8a390d0 --- /dev/null +++ b/plugins/catalog-backend/src/util/readScmEventHandlingConfig.ts @@ -0,0 +1,56 @@ +/* + * Copyright 2026 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import lodash from 'lodash'; + +export interface ScmEventHandlingConfig { + refresh: boolean; + unregister: boolean; + move: boolean; +} + +const disabled: ScmEventHandlingConfig = { + refresh: false, + unregister: false, + move: false, +}; + +const defaults: ScmEventHandlingConfig = { + refresh: true, + unregister: true, + move: true, +}; + +export function readScmEventHandlingConfig( + config: Config, +): ScmEventHandlingConfig { + const rootKey = 'catalog.scmEvents'; + + if (!config.has(rootKey) || config.get(rootKey) === true) { + return { ...defaults }; + } else if (config.get(rootKey) === false) { + return { ...disabled }; + } + + const given = { + refresh: config.getOptionalBoolean(`${rootKey}.refresh`), + unregister: config.getOptionalBoolean(`${rootKey}.unregister`), + move: config.getOptionalBoolean(`${rootKey}.move`), + }; + + return lodash.merge({}, defaults, given); +} diff --git a/plugins/catalog-node/report-alpha.api.md b/plugins/catalog-node/report-alpha.api.md index 2632184e46..aad7518921 100644 --- a/plugins/catalog-node/report-alpha.api.md +++ b/plugins/catalog-node/report-alpha.api.md @@ -134,7 +134,7 @@ export interface CatalogScmEventsService { // @alpha export const catalogScmEventsServiceRef: ServiceRef< CatalogScmEventsService, - 'root', + 'plugin', 'singleton' >; diff --git a/plugins/catalog-node/src/scmEvents/catalogScmEventsServiceRef.ts b/plugins/catalog-node/src/scmEvents/catalogScmEventsServiceRef.ts index 631d5b80c1..5a2791eace 100644 --- a/plugins/catalog-node/src/scmEvents/catalogScmEventsServiceRef.ts +++ b/plugins/catalog-node/src/scmEvents/catalogScmEventsServiceRef.ts @@ -33,14 +33,16 @@ import { DefaultCatalogScmEventsService } from './DefaultCatalogScmEventsService */ export const catalogScmEventsServiceRef = createServiceRef({ - id: 'catalog-scm-events', - scope: 'root', + id: 'catalog.scm-events.alpha', defaultFactory: async service => createServiceFactory({ service, deps: {}, - factory() { + createRootContext() { return new DefaultCatalogScmEventsService(); }, + factory(_, ctx) { + return ctx; + }, }), });