diff --git a/.changeset/itchy-moose-raise.md b/.changeset/itchy-moose-raise.md new file mode 100644 index 0000000000..7e90f729e7 --- /dev/null +++ b/.changeset/itchy-moose-raise.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added opt-in ability to evict entities from the catalog whose provider is no longer configured. See [Catalog configuration documentation](https://backstage.io/docs/features/software-catalog/configuration#clean-up-entities-from-orphaned-entity-providers) diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md index 6ed5a1d4b6..f8de3034fd 100644 --- a/docs/features/software-catalog/configuration.md +++ b/docs/features/software-catalog/configuration.md @@ -163,6 +163,17 @@ catalog: orphanStrategy: delete ``` +## Clean up entities from orphaned entity providers + +By default, if an entity provider which has provided entities to the catalog, is no longer configured, then the entities remain in the catalog until they are manually unregistered. + +To remove these entities automatically, you can use the following configuration. + +```yaml +catalog: + orphanProviderStrategy: delete +``` + ## Processing Interval The [processing loop](./life-of-an-entity.md#processing) is diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index a005e7e81a..d7816fa856 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -155,6 +155,13 @@ export interface Config { */ orphanStrategy?: 'keep' | 'delete'; + /** + * The strategy to use for entities that are referenced by providers that are orphaned, + * i.e. entities with no providers currently configured in the catalog. The default value is + * "keep". + */ + orphanProviderStrategy?: 'keep' | 'delete'; + /** * The strategy to use when stitching together the final entities. */ diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts index ac361795e1..383e06100f 100644 --- a/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts @@ -59,21 +59,21 @@ describe('DefaultProviderDatabase', () => { await db('refresh_state').insert(ref); }; - describe('replaceUnprocessedEntities', () => { - const createLocations = async (db: Knex, entityRefs: string[]) => { - for (const ref of entityRefs) { - await insertRefreshStateRow(db, { - entity_id: uuid.v4(), - entity_ref: ref, - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: '2021-04-01 13:37:00', - last_discovery_at: '2021-04-01 13:37:00', - }); - } - }; + const createLocations = async (db: Knex, entityRefs: string[]) => { + for (const ref of entityRefs) { + await insertRefreshStateRow(db, { + entity_id: uuid.v4(), + entity_ref: ref, + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + } + }; + describe('replaceUnprocessedEntities', () => { it.each(databases.eachSupportedId())( 'replaces all existing state correctly for simple dependency chains, %p', async databaseId => { @@ -988,4 +988,86 @@ describe('DefaultProviderDatabase', () => { }, ); }); + + describe('listReferenceSourceKeys', () => { + it.each(databases.eachSupportedId())( + 'returns the source_keys from "refresh_state_references", %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + + await createLocations(knex, [ + 'location:default/root', + 'location:default/root-1', + ]); + + await insertRefRow(knex, { + source_key: 'foo', + target_entity_ref: 'location:default/root', + }); + await insertRefRow(knex, { + source_key: 'bar', + target_entity_ref: 'location:default/root-1', + }); + + const res = await db.transaction(async tx => + db.listReferenceSourceKeys(tx), + ); + + expect(res).toEqual(['bar', 'foo']); + }, + ); + + it.each(databases.eachSupportedId())( + 'returns only unique source_keys", %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + + await createLocations(knex, [ + 'location:default/root', + 'location:default/root-1', + ]); + + await insertRefRow(knex, { + source_key: 'foo', + target_entity_ref: 'location:default/root', + }); + await insertRefRow(knex, { + source_key: 'foo', + target_entity_ref: 'location:default/root-1', + }); + + const res = await db.transaction(async tx => + db.listReferenceSourceKeys(tx), + ); + + expect(res).toEqual(['foo']); + }, + ); + + it.each(databases.eachSupportedId())( + 'does not return null source_keys", %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + + await createLocations(knex, [ + 'location:default/root', + 'location:default/root-1', + ]); + + await insertRefRow(knex, { + source_key: 'foo', + target_entity_ref: 'location:default/root', + }); + await insertRefRow(knex, { + target_entity_ref: 'location:default/root-1', + }); + + const res = await db.transaction(async tx => + db.listReferenceSourceKeys(tx), + ); + + expect(res).toEqual(['foo']); + }, + ); + }); }); diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts index 60dd43bb59..958e1b0f2d 100644 --- a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts @@ -199,6 +199,20 @@ export class DefaultProviderDatabase implements ProviderDatabase { } } + async listReferenceSourceKeys(txOpaque: Transaction): Promise { + const tx = txOpaque as Knex | Knex.Transaction; + + const rows = await tx( + 'refresh_state_references', + ) + .distinct('source_key') + .whereNotNull('source_key'); + + return rows + .map(row => row.source_key) + .filter((key): key is string => !!key); + } + async refreshByRefreshKeys( txOpaque: Transaction, options: RefreshByKeyOptions, diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 82a1ac3f6b..88003915bd 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -173,6 +173,11 @@ export interface ProviderDatabase { txOpaque: Transaction, options: RefreshByKeyOptions, ): Promise; + + /** + * List the names of all the entity providers that have references in the provider database. + */ + listReferenceSourceKeys(txOpaque: Transaction): Promise; } // TODO(Rugvip): This is only partial for now diff --git a/plugins/catalog-backend/src/processing/evictEntitiesFromOrphanedProviders.test.ts b/plugins/catalog-backend/src/processing/evictEntitiesFromOrphanedProviders.test.ts new file mode 100644 index 0000000000..1580194639 --- /dev/null +++ b/plugins/catalog-backend/src/processing/evictEntitiesFromOrphanedProviders.test.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EntityProvider } from '@backstage/plugin-catalog-node'; +import { mockServices } from '@backstage/backend-test-utils'; +import { DefaultProviderDatabase } from '../database/DefaultProviderDatabase'; +import { evictEntitiesFromOrphanedProviders } from './evictEntitiesFromOrphanedProviders'; + +describe('evictEntitiesFromOrphanedProviders', () => { + const db = { + transaction: jest.fn().mockImplementation(cb => cb((() => {}) as any)), + replaceUnprocessedEntities: jest.fn(), + listReferenceSourceKeys: jest.fn(), + } as unknown as jest.Mocked; + + const providers = [ + { getProviderName: () => 'provider1' }, + { getProviderName: () => 'provider2' }, + ] as unknown as EntityProvider[]; + const logger = mockServices.logger.mock(); + + it('replaces unprocessed entities for orphaned providers with empty items', async () => { + db.listReferenceSourceKeys.mockResolvedValue(['foo', 'bar']); + + await evictEntitiesFromOrphanedProviders({ db, providers, logger }); + + expect(db.replaceUnprocessedEntities).toHaveBeenCalledTimes(2); + expect(db.replaceUnprocessedEntities).toHaveBeenNthCalledWith( + 1, + expect.anything(), + { + sourceKey: 'foo', + type: 'full', + items: [], + }, + ); + expect(db.replaceUnprocessedEntities).toHaveBeenNthCalledWith( + 2, + expect.anything(), + { + sourceKey: 'bar', + type: 'full', + items: [], + }, + ); + }); + + it('does not replace unprocessed entities for providers that are not orphaned', async () => { + db.listReferenceSourceKeys.mockResolvedValue(['foo', 'provider1']); + + await evictEntitiesFromOrphanedProviders({ db, providers, logger }); + + expect(db.replaceUnprocessedEntities).not.toHaveBeenCalledWith( + expect.anything(), + { + sourceKey: 'provider1', + type: 'full', + items: [], + }, + ); + }); +}); diff --git a/plugins/catalog-backend/src/processing/evictEntitiesFromOrphanedProviders.ts b/plugins/catalog-backend/src/processing/evictEntitiesFromOrphanedProviders.ts new file mode 100644 index 0000000000..a6bda5e8c5 --- /dev/null +++ b/plugins/catalog-backend/src/processing/evictEntitiesFromOrphanedProviders.ts @@ -0,0 +1,77 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityProvider } from '@backstage/plugin-catalog-node'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { ProviderDatabase } from '../database/types'; + +async function getOrphanedEntityProviderNames({ + db, + providers, +}: { + db: ProviderDatabase; + providers: EntityProvider[]; +}): Promise { + const dbProviderNames = await db.transaction(async tx => + db.listReferenceSourceKeys(tx), + ); + + const providerNames = providers.map(p => p.getProviderName()); + + return dbProviderNames.filter( + dbProviderName => !providerNames.includes(dbProviderName), + ); +} + +async function removeEntitiesForProvider({ + db, + providerName, + logger, +}: { + db: ProviderDatabase; + providerName: string; + logger: LoggerService; +}) { + try { + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + sourceKey: providerName, + type: 'full', + items: [], + }); + }); + logger.info(`Removed entities for orphaned provider ${providerName}`); + } catch (e) { + logger.error( + `Failed to remove entities for orphaned provider ${providerName}`, + e, + ); + } +} + +export async function evictEntitiesFromOrphanedProviders(options: { + db: ProviderDatabase; + providers: EntityProvider[]; + logger: LoggerService; +}) { + for (const providerName of await getOrphanedEntityProviderNames(options)) { + await removeEntitiesForProvider({ + db: options.db, + providerName, + logger: options.logger, + }); + } +} diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 79121c86c5..386d6cf9cd 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -87,6 +87,7 @@ import { ProcessingIntervalFunction, } from '../processing'; import { connectEntityProviders } from '../processing/connectEntityProviders'; +import { evictEntitiesFromOrphanedProviders } from '../processing/evictEntitiesFromOrphanedProviders'; import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine'; import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; import { @@ -643,6 +644,15 @@ export class CatalogBuilder { disableRelationsCompatibility, }); + if ( + config.getOptionalString('catalog.orphanProviderStrategy') === 'delete' + ) { + await evictEntitiesFromOrphanedProviders({ + db: providerDatabase, + providers: entityProviders, + logger, + }); + } await connectEntityProviders(providerDatabase, entityProviders); return {