From 259b22efddecc5f4f7ec819887e30342ff00bab4 Mon Sep 17 00:00:00 2001 From: Jack Palmer Date: Tue, 25 Feb 2025 17:02:04 +0000 Subject: [PATCH] feat: Add option to evict entities belonging to an orphaned entity provider Signed-off-by: Jack Palmer --- plugins/catalog-backend/config.d.ts | 8 +++ .../src/database/DefaultProviderDatabase.ts | 14 ++++ plugins/catalog-backend/src/database/types.ts | 5 ++ .../evictOrphanedEntityProviders.ts | 69 +++++++++++++++++++ .../src/service/CatalogBuilder.ts | 9 +++ 5 files changed, 105 insertions(+) create mode 100644 plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index a005e7e81a..6c6bc95406 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -216,5 +216,13 @@ export interface Config { * This flag is temporary and will be enabled by default in future releases. */ useUrlReadersSearch?: boolean; + + /** + * Evicts entities from the catalog that are no longer referenced by any + * added entity providers. + * + * Defaults to false. + */ + evictOrphanedEntityProviders?: boolean; }; } diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts index 60dd43bb59..25c276bbb1 100644 --- a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts @@ -199,6 +199,20 @@ export class DefaultProviderDatabase implements ProviderDatabase { } } + async listEntityProviderNames(txOpaque: Transaction): Promise { + const tx = txOpaque as Knex | Knex.Transaction; + + const rows = await tx( + 'refresh_state_references', + ) + .distinct('source_key') + .whereNotNull('source_key'); + + return rows + .map(row => row.source_key) + .filter((key): key is string => !!key); + } + async refreshByRefreshKeys( txOpaque: Transaction, options: RefreshByKeyOptions, diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 82a1ac3f6b..5f8e9d5cdc 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -173,6 +173,11 @@ export interface ProviderDatabase { txOpaque: Transaction, options: RefreshByKeyOptions, ): Promise; + + /** + * List the names of all the entity providers that have references in the provider database. + */ + listEntityProviderNames(txOpaque: Transaction): Promise; } // TODO(Rugvip): This is only partial for now diff --git a/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts b/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts new file mode 100644 index 0000000000..7a53309f5e --- /dev/null +++ b/plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts @@ -0,0 +1,69 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityProvider } from '@backstage/plugin-catalog-node'; +import { ProviderDatabase } from '../database/types'; +import { LoggerService } from '@backstage/backend-plugin-api'; + +async function getOrphanedEntityProviderNames( + db: ProviderDatabase, + providers: EntityProvider[], +): Promise { + const dbProviderNames = await db.transaction(async tx => + db.listEntityProviderNames(tx), + ); + + const providerNames = providers.map(p => p.getProviderName()); + + return dbProviderNames.filter( + dbProviderName => !providerNames.includes(dbProviderName), + ); +} + +async function removeEntitiesForProvider( + db: ProviderDatabase, + providerName: string, + logger: LoggerService, +) { + try { + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + sourceKey: providerName, + type: 'full', + items: [], + }); + }); + logger.info(`Removed entities for orphaned provider ${providerName}`); + } catch (e) { + logger.error( + `Failed to remove entities for orphaned provider ${providerName}`, + e, + ); + } +} + +export async function evictOrphanedEntityProviders( + db: ProviderDatabase, + providers: EntityProvider[], + logger: LoggerService, +) { + for (const providerName of await getOrphanedEntityProviderNames( + db, + providers, + )) { + await removeEntitiesForProvider(db, providerName, logger); + } +} diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 79121c86c5..9f98a8fa0f 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -87,6 +87,7 @@ import { ProcessingIntervalFunction, } from '../processing'; import { connectEntityProviders } from '../processing/connectEntityProviders'; +import { evictOrphanedEntityProviders } from '../processing/evictOrphanedEntityProviders'; import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine'; import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; import { @@ -645,6 +646,14 @@ export class CatalogBuilder { await connectEntityProviders(providerDatabase, entityProviders); + if (config.getOptionalBoolean('catalog.evictOrphanedEntityProviders')) { + await evictOrphanedEntityProviders( + providerDatabase, + entityProviders, + logger, + ); + } + return { processingEngine: { async start() {