From 259b22efddecc5f4f7ec819887e30342ff00bab4 Mon Sep 17 00:00:00 2001 From: Jack Palmer Date: Tue, 25 Feb 2025 17:02:04 +0000 Subject: [PATCH 1/7] feat: Add option to evict entities belonging to an orphaned entity provider Signed-off-by: Jack Palmer --- plugins/catalog-backend/config.d.ts | 8 +++ .../src/database/DefaultProviderDatabase.ts | 14 ++++ plugins/catalog-backend/src/database/types.ts | 5 ++ .../evictOrphanedEntityProviders.ts | 69 +++++++++++++++++++ .../src/service/CatalogBuilder.ts | 9 +++ 5 files changed, 105 insertions(+) create mode 100644 plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index a005e7e81a..6c6bc95406 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -216,5 +216,13 @@ export interface Config { * This flag is temporary and will be enabled by default in future releases. */ useUrlReadersSearch?: boolean; + + /** + * Evicts entities from the catalog that are no longer referenced by any + * added entity providers. + * + * Defaults to false. + */ + evictOrphanedEntityProviders?: boolean; }; } diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts index 60dd43bb59..25c276bbb1 100644 --- a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts @@ -199,6 +199,20 @@ export class DefaultProviderDatabase implements ProviderDatabase { } } + async listEntityProviderNames(txOpaque: Transaction): Promise { + const tx = txOpaque as Knex | Knex.Transaction; + + const rows = await tx( + 'refresh_state_references', + ) + .distinct('source_key') + .whereNotNull('source_key'); + + return rows + .map(row => row.source_key) + .filter((key): key is string => !!key); + } + async refreshByRefreshKeys( txOpaque: Transaction, options: RefreshByKeyOptions, diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 82a1ac3f6b..5f8e9d5cdc 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -173,6 +173,11 @@ export interface ProviderDatabase { txOpaque: Transaction, options: RefreshByKeyOptions, ): Promise; + + /** + * List the names of all the entity providers that have references in the provider database. + */ + listEntityProviderNames(txOpaque: Transaction): Promise; } // TODO(Rugvip): This is only partial for now diff --git a/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts b/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts new file mode 100644 index 0000000000..7a53309f5e --- /dev/null +++ b/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts @@ -0,0 +1,69 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityProvider } from '@backstage/plugin-catalog-node'; +import { ProviderDatabase } from '../database/types'; +import { LoggerService } from '@backstage/backend-plugin-api'; + +async function getOrphanedEntityProviderNames( + db: ProviderDatabase, + providers: EntityProvider[], +): Promise { + const dbProviderNames = await db.transaction(async tx => + db.listEntityProviderNames(tx), + ); + + const providerNames = providers.map(p => p.getProviderName()); + + return dbProviderNames.filter( + dbProviderName => !providerNames.includes(dbProviderName), + ); +} + +async function removeEntitiesForProvider( + db: ProviderDatabase, + providerName: string, + logger: LoggerService, +) { + try { + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + sourceKey: providerName, + type: 'full', + items: [], + }); + }); + logger.info(`Removed entities for orphaned provider ${providerName}`); + } catch (e) { + logger.error( + `Failed to remove entities for orphaned provider ${providerName}`, + e, + ); + } +} + +export async function evictOrphanedEntityProviders( + db: ProviderDatabase, + providers: EntityProvider[], + logger: LoggerService, +) { + for (const providerName of await getOrphanedEntityProviderNames( + db, + providers, + )) { + await removeEntitiesForProvider(db, providerName, logger); + } +} diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 79121c86c5..9f98a8fa0f 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -87,6 +87,7 @@ import { ProcessingIntervalFunction, } from '../processing'; import { connectEntityProviders } from '../processing/connectEntityProviders'; +import { evictOrphanedEntityProviders } from '../processing/evictOrphanedEntityProviders'; import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine'; import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; import { @@ -645,6 +646,14 @@ export class CatalogBuilder { await connectEntityProviders(providerDatabase, entityProviders); + if (config.getOptionalBoolean('catalog.evictOrphanedEntityProviders')) { + await evictOrphanedEntityProviders( + providerDatabase, + entityProviders, + logger, + ); + } + return { processingEngine: { async start() { From 1dc906c10a07a669e162cae0cb1b02fccf091a36 Mon Sep 17 00:00:00 2001 From: Jack Palmer Date: Tue, 25 Feb 2025 20:10:40 +0000 Subject: [PATCH 2/7] chore: Add tests for ProviderDatabase and evictOrphanedEntityProviders Signed-off-by: Jack Palmer --- plugins/catalog-backend/config.d.ts | 5 +- .../database/DefaultProviderDatabase.test.ts | 110 +++++++++++++++--- .../src/database/DefaultProviderDatabase.ts | 2 +- plugins/catalog-backend/src/database/types.ts | 2 +- .../evictOrphanedEntityProviders.test.ts | 74 ++++++++++++ .../evictOrphanedEntityProviders.ts | 50 ++++---- .../src/service/CatalogBuilder.ts | 8 +- 7 files changed, 206 insertions(+), 45 deletions(-) create mode 100644 plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.test.ts diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 6c6bc95406..1169b4ccfa 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -218,10 +218,7 @@ export interface Config { useUrlReadersSearch?: boolean; /** - * Evicts entities from the catalog that are no longer referenced by any - * added entity providers. - * - * Defaults to false. + * Evicts entities from the catalog that are no longer referenced by entity providers added to the catalog. */ evictOrphanedEntityProviders?: boolean; }; diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts index ac361795e1..383e06100f 100644 --- a/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts @@ -59,21 +59,21 @@ describe('DefaultProviderDatabase', () => { await db('refresh_state').insert(ref); }; - describe('replaceUnprocessedEntities', () => { - const createLocations = async (db: Knex, entityRefs: string[]) => { - for (const ref of entityRefs) { - await insertRefreshStateRow(db, { - entity_id: uuid.v4(), - entity_ref: ref, - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: '2021-04-01 13:37:00', - last_discovery_at: '2021-04-01 13:37:00', - }); - } - }; + const createLocations = async (db: Knex, entityRefs: string[]) => { + for (const ref of entityRefs) { + await insertRefreshStateRow(db, { + entity_id: uuid.v4(), + entity_ref: ref, + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + } + }; + describe('replaceUnprocessedEntities', () => { it.each(databases.eachSupportedId())( 'replaces all existing state correctly for simple dependency chains, %p', async databaseId => { @@ -988,4 +988,86 @@ describe('DefaultProviderDatabase', () => { }, ); }); + + describe('listReferenceSourceKeys', () => { + it.each(databases.eachSupportedId())( + 'returns the source_keys from "refresh_state_references", %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + + await createLocations(knex, [ + 'location:default/root', + 'location:default/root-1', + ]); + + await insertRefRow(knex, { + source_key: 'foo', + target_entity_ref: 'location:default/root', + }); + await insertRefRow(knex, { + source_key: 'bar', + target_entity_ref: 'location:default/root-1', + }); + + const res = await db.transaction(async tx => + db.listReferenceSourceKeys(tx), + ); + + expect(res).toEqual(['bar', 'foo']); + }, + ); + + it.each(databases.eachSupportedId())( + 'returns only unique source_keys", %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + + await createLocations(knex, [ + 'location:default/root', + 'location:default/root-1', + ]); + + await insertRefRow(knex, { + source_key: 'foo', + target_entity_ref: 'location:default/root', + }); + await insertRefRow(knex, { + source_key: 'foo', + target_entity_ref: 'location:default/root-1', + }); + + const res = await db.transaction(async tx => + db.listReferenceSourceKeys(tx), + ); + + expect(res).toEqual(['foo']); + }, + ); + + it.each(databases.eachSupportedId())( + 'does not return null source_keys", %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + + await createLocations(knex, [ + 'location:default/root', + 'location:default/root-1', + ]); + + await insertRefRow(knex, { + source_key: 'foo', + target_entity_ref: 'location:default/root', + }); + await insertRefRow(knex, { + target_entity_ref: 'location:default/root-1', + }); + + const res = await db.transaction(async tx => + db.listReferenceSourceKeys(tx), + ); + + expect(res).toEqual(['foo']); + }, + ); + }); }); diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts index 25c276bbb1..958e1b0f2d 100644 --- a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts @@ -199,7 +199,7 @@ export class DefaultProviderDatabase implements ProviderDatabase { } } - async listEntityProviderNames(txOpaque: Transaction): Promise { + async listReferenceSourceKeys(txOpaque: Transaction): Promise { const tx = txOpaque as Knex | Knex.Transaction; const rows = await tx( diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 5f8e9d5cdc..88003915bd 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -177,7 +177,7 @@ export interface ProviderDatabase { /** * List the names of all the entity providers that have references in the provider database. */ - listEntityProviderNames(txOpaque: Transaction): Promise; + listReferenceSourceKeys(txOpaque: Transaction): Promise; } // TODO(Rugvip): This is only partial for now diff --git a/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.test.ts b/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.test.ts new file mode 100644 index 0000000000..ba618feaba --- /dev/null +++ b/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.test.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EntityProvider } from '@backstage/plugin-catalog-node'; +import { mockServices } from '@backstage/backend-test-utils'; +import { DefaultProviderDatabase } from '../database/DefaultProviderDatabase'; +import { evictOrphanedEntityProviders } from './evictOrphanedEntityProviders'; + +describe('evictOrphanedEntityProviders', () => { + const db = { + transaction: jest.fn().mockImplementation(cb => cb((() => {}) as any)), + replaceUnprocessedEntities: jest.fn(), + listReferenceSourceKeys: jest.fn(), + } as unknown as jest.Mocked; + + const providers = [ + { getProviderName: () => 'provider1' }, + { getProviderName: () => 'provider2' }, + ] as unknown as EntityProvider[]; + const logger = mockServices.logger.mock(); + + it('replaces unprocessed entities for orphaned providers with empty items', async () => { + db.listReferenceSourceKeys.mockResolvedValue(['foo', 'bar']); + + await evictOrphanedEntityProviders({ db, providers, logger }); + + expect(db.replaceUnprocessedEntities).toHaveBeenCalledTimes(2); + expect(db.replaceUnprocessedEntities).toHaveBeenNthCalledWith( + 1, + expect.anything(), + { + sourceKey: 'foo', + type: 'full', + items: [], + }, + ); + expect(db.replaceUnprocessedEntities).toHaveBeenNthCalledWith( + 2, + expect.anything(), + { + sourceKey: 'bar', + type: 'full', + items: [], + }, + ); + }); + + it('does not replace unprocessed entities for providers that are not orphaned', async () => { + db.listReferenceSourceKeys.mockResolvedValue(['foo', 'provider1']); + + await evictOrphanedEntityProviders({ db, providers, logger }); + + expect(db.replaceUnprocessedEntities).not.toHaveBeenCalledWith( + expect.anything(), + { + sourceKey: 'provider1', + type: 'full', + items: [], + }, + ); + }); +}); diff --git a/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts b/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts index 7a53309f5e..f1d99e83d7 100644 --- a/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts +++ b/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts @@ -15,15 +15,18 @@ */ import { EntityProvider } from '@backstage/plugin-catalog-node'; -import { ProviderDatabase } from '../database/types'; import { LoggerService } from '@backstage/backend-plugin-api'; +import { ProviderDatabase } from '../database/types'; -async function getOrphanedEntityProviderNames( - db: ProviderDatabase, - providers: EntityProvider[], -): Promise { +async function getOrphanedEntityProviderNames({ + db, + providers, +}: { + db: ProviderDatabase; + providers: EntityProvider[]; +}): Promise { const dbProviderNames = await db.transaction(async tx => - db.listEntityProviderNames(tx), + db.listReferenceSourceKeys(tx), ); const providerNames = providers.map(p => p.getProviderName()); @@ -33,11 +36,15 @@ async function getOrphanedEntityProviderNames( ); } -async function removeEntitiesForProvider( - db: ProviderDatabase, - providerName: string, - logger: LoggerService, -) { +async function removeEntitiesForProvider({ + db, + providerName, + logger, +}: { + db: ProviderDatabase; + providerName: string; + logger: LoggerService; +}) { try { await db.transaction(async tx => { await db.replaceUnprocessedEntities(tx, { @@ -55,15 +62,16 @@ async function removeEntitiesForProvider( } } -export async function evictOrphanedEntityProviders( - db: ProviderDatabase, - providers: EntityProvider[], - logger: LoggerService, -) { - for (const providerName of await getOrphanedEntityProviderNames( - db, - providers, - )) { - await removeEntitiesForProvider(db, providerName, logger); +export async function evictOrphanedEntityProviders(options: { + db: ProviderDatabase; + providers: EntityProvider[]; + logger: LoggerService; +}) { + for (const providerName of await getOrphanedEntityProviderNames(options)) { + await removeEntitiesForProvider({ + db: options.db, + providerName, + logger: options.logger, + }); } } diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 9f98a8fa0f..dd77d70efd 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -647,11 +647,11 @@ export class CatalogBuilder { await connectEntityProviders(providerDatabase, entityProviders); if (config.getOptionalBoolean('catalog.evictOrphanedEntityProviders')) { - await evictOrphanedEntityProviders( - providerDatabase, - entityProviders, + await evictOrphanedEntityProviders({ + db: providerDatabase, + providers: entityProviders, logger, - ); + }); } return { From f95ef03dff5b6543874d073c196b28333a73c8dd Mon Sep 17 00:00:00 2001 From: Jack Palmer Date: Tue, 25 Feb 2025 20:22:11 +0000 Subject: [PATCH 3/7] chore: changeset Signed-off-by: Jack Palmer --- .changeset/young-carpets-love.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/young-carpets-love.md diff --git a/.changeset/young-carpets-love.md b/.changeset/young-carpets-love.md new file mode 100644 index 0000000000..3c16a0a7b2 --- /dev/null +++ b/.changeset/young-carpets-love.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Added opt-in ability to evict entities from the catalog whose provider is no longer configured From da04ead6beac93d697e116bc2571242de1e5f3b4 Mon Sep 17 00:00:00 2001 From: Jack Palmer Date: Tue, 25 Feb 2025 20:31:02 +0000 Subject: [PATCH 4/7] chore: docs Signed-off-by: Jack Palmer --- docs/features/software-catalog/configuration.md | 11 +++++++++++ plugins/catalog-backend/config.d.ts | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md index 6ed5a1d4b6..df252808eb 100644 --- a/docs/features/software-catalog/configuration.md +++ b/docs/features/software-catalog/configuration.md @@ -163,6 +163,17 @@ catalog: orphanStrategy: delete ``` +## Clean up entities from orphaned entity providers + +By default, if an entity provider which has provided entities to the catalog, is no longer configured, then the entities remain in the catalog until they are manually unregistered. + +To remove these entities automatically, you can use the following configuration. + +```yaml +catalog: + evictOrphanedEntityProviders: true +``` + ## Processing Interval The [processing loop](./life-of-an-entity.md#processing) is diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 1169b4ccfa..057d581e80 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -218,7 +218,7 @@ export interface Config { useUrlReadersSearch?: boolean; /** - * Evicts entities from the catalog that are no longer referenced by entity providers added to the catalog. + * Evicts entities from the catalog when their related entity provider no longer exists. */ evictOrphanedEntityProviders?: boolean; }; From 077493b896e45a48c42b66951891b761e178a92d Mon Sep 17 00:00:00 2001 From: Jack Palmer Date: Wed, 26 Feb 2025 09:45:29 +0000 Subject: [PATCH 5/7] refactor: Change config option to be consistent with orphanStrategy Signed-off-by: Jack Palmer --- docs/features/software-catalog/configuration.md | 2 +- plugins/catalog-backend/config.d.ts | 12 +++++++----- ...ts => evictEntitiesFromOrphanedProviders.test.ts} | 8 ++++---- ...ders.ts => evictEntitiesFromOrphanedProviders.ts} | 2 +- .../catalog-backend/src/service/CatalogBuilder.ts | 8 +++++--- 5 files changed, 18 insertions(+), 14 deletions(-) rename plugins/catalog-backend/src/processing/{evictOrphanedEntityProviders.test.ts => evictEntitiesFromOrphanedProviders.test.ts} (88%) rename plugins/catalog-backend/src/processing/{evictOrphanedEntityProviders.ts => evictEntitiesFromOrphanedProviders.ts} (96%) diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md index df252808eb..f8de3034fd 100644 --- a/docs/features/software-catalog/configuration.md +++ b/docs/features/software-catalog/configuration.md @@ -171,7 +171,7 @@ To remove these entities automatically, you can use the following configuration. ```yaml catalog: - evictOrphanedEntityProviders: true + orphanProviderStrategy: delete ``` ## Processing Interval diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 057d581e80..d7816fa856 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -155,6 +155,13 @@ export interface Config { */ orphanStrategy?: 'keep' | 'delete'; + /** + * The strategy to use for entities that are referenced by providers that are orphaned, + * i.e. entities with no providers currently configured in the catalog. The default value is + * "keep". + */ + orphanProviderStrategy?: 'keep' | 'delete'; + /** * The strategy to use when stitching together the final entities. */ @@ -216,10 +223,5 @@ export interface Config { * This flag is temporary and will be enabled by default in future releases. */ useUrlReadersSearch?: boolean; - - /** - * Evicts entities from the catalog when their related entity provider no longer exists. - */ - evictOrphanedEntityProviders?: boolean; }; } diff --git a/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.test.ts b/plugins/catalog-backend/src/processing/evictEntitiesFromOrphanedProviders.test.ts similarity index 88% rename from plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.test.ts rename to plugins/catalog-backend/src/processing/evictEntitiesFromOrphanedProviders.test.ts index ba618feaba..1580194639 100644 --- a/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.test.ts +++ b/plugins/catalog-backend/src/processing/evictEntitiesFromOrphanedProviders.test.ts @@ -16,9 +16,9 @@ import { EntityProvider } from '@backstage/plugin-catalog-node'; import { mockServices } from '@backstage/backend-test-utils'; import { DefaultProviderDatabase } from '../database/DefaultProviderDatabase'; -import { evictOrphanedEntityProviders } from './evictOrphanedEntityProviders'; +import { evictEntitiesFromOrphanedProviders } from './evictEntitiesFromOrphanedProviders'; -describe('evictOrphanedEntityProviders', () => { +describe('evictEntitiesFromOrphanedProviders', () => { const db = { transaction: jest.fn().mockImplementation(cb => cb((() => {}) as any)), replaceUnprocessedEntities: jest.fn(), @@ -34,7 +34,7 @@ describe('evictOrphanedEntityProviders', () => { it('replaces unprocessed entities for orphaned providers with empty items', async () => { db.listReferenceSourceKeys.mockResolvedValue(['foo', 'bar']); - await evictOrphanedEntityProviders({ db, providers, logger }); + await evictEntitiesFromOrphanedProviders({ db, providers, logger }); expect(db.replaceUnprocessedEntities).toHaveBeenCalledTimes(2); expect(db.replaceUnprocessedEntities).toHaveBeenNthCalledWith( @@ -60,7 +60,7 @@ describe('evictOrphanedEntityProviders', () => { it('does not replace unprocessed entities for providers that are not orphaned', async () => { db.listReferenceSourceKeys.mockResolvedValue(['foo', 'provider1']); - await evictOrphanedEntityProviders({ db, providers, logger }); + await evictEntitiesFromOrphanedProviders({ db, providers, logger }); expect(db.replaceUnprocessedEntities).not.toHaveBeenCalledWith( expect.anything(), diff --git a/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts b/plugins/catalog-backend/src/processing/evictEntitiesFromOrphanedProviders.ts similarity index 96% rename from plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts rename to plugins/catalog-backend/src/processing/evictEntitiesFromOrphanedProviders.ts index f1d99e83d7..a6bda5e8c5 100644 --- a/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts +++ b/plugins/catalog-backend/src/processing/evictEntitiesFromOrphanedProviders.ts @@ -62,7 +62,7 @@ async function removeEntitiesForProvider({ } } -export async function evictOrphanedEntityProviders(options: { +export async function evictEntitiesFromOrphanedProviders(options: { db: ProviderDatabase; providers: EntityProvider[]; logger: LoggerService; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index dd77d70efd..0965ba63e3 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -87,7 +87,7 @@ import { ProcessingIntervalFunction, } from '../processing'; import { connectEntityProviders } from '../processing/connectEntityProviders'; -import { evictOrphanedEntityProviders } from '../processing/evictOrphanedEntityProviders'; +import { evictEntitiesFromOrphanedProviders } from '../processing/evictEntitiesFromOrphanedProviders'; import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine'; import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; import { @@ -646,8 +646,10 @@ export class CatalogBuilder { await connectEntityProviders(providerDatabase, entityProviders); - if (config.getOptionalBoolean('catalog.evictOrphanedEntityProviders')) { - await evictOrphanedEntityProviders({ + if ( + config.getOptionalString('catalog.orphanProviderStrategy') === 'delete' + ) { + await evictEntitiesFromOrphanedProviders({ db: providerDatabase, providers: entityProviders, logger, From ca9c51ba1bb30ea326e323c37e1ba469222a2452 Mon Sep 17 00:00:00 2001 From: Jack Palmer Date: Wed, 26 Feb 2025 09:50:49 +0000 Subject: [PATCH 6/7] chore: Update changeset to minor Signed-off-by: Jack Palmer --- .changeset/itchy-moose-raise.md | 5 +++++ .changeset/young-carpets-love.md | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 .changeset/itchy-moose-raise.md delete mode 100644 .changeset/young-carpets-love.md diff --git a/.changeset/itchy-moose-raise.md b/.changeset/itchy-moose-raise.md new file mode 100644 index 0000000000..7e90f729e7 --- /dev/null +++ b/.changeset/itchy-moose-raise.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added opt-in ability to evict entities from the catalog whose provider is no longer configured. See [Catalog configuration documentation](https://backstage.io/docs/features/software-catalog/configuration#clean-up-entities-from-orphaned-entity-providers) diff --git a/.changeset/young-carpets-love.md b/.changeset/young-carpets-love.md deleted file mode 100644 index 3c16a0a7b2..0000000000 --- a/.changeset/young-carpets-love.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-catalog-backend': patch ---- - -Added opt-in ability to evict entities from the catalog whose provider is no longer configured From abd0ab7901aff9539992c03e17764acd8622fc61 Mon Sep 17 00:00:00 2001 From: Jack Palmer Date: Wed, 26 Feb 2025 17:47:45 +0000 Subject: [PATCH 7/7] feat: remove entities before connecting entity providers Signed-off-by: Jack Palmer --- plugins/catalog-backend/src/service/CatalogBuilder.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 0965ba63e3..386d6cf9cd 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -644,8 +644,6 @@ export class CatalogBuilder { disableRelationsCompatibility, }); - await connectEntityProviders(providerDatabase, entityProviders); - if ( config.getOptionalString('catalog.orphanProviderStrategy') === 'delete' ) { @@ -655,6 +653,7 @@ export class CatalogBuilder { logger, }); } + await connectEntityProviders(providerDatabase, entityProviders); return { processingEngine: {