feat: Add option to evict entities belonging to an orphaned entity provider

Signed-off-by: Jack Palmer <jackpalmer@spotify.com>

plugins/catalog-backend/config.d.ts

@@ -216,5 +216,13 @@ export interface Config {
      * This flag is temporary and will be enabled by default in future releases.
      */
     useUrlReadersSearch?: boolean;
+
+    /**
+     * Evicts entities from the catalog that are no longer referenced by any
+     * added entity providers.
+     *
+     * Defaults to false.
+     */
+    evictOrphanedEntityProviders?: boolean;
   };
 }

plugins/catalog-backend/src/database/DefaultProviderDatabase.ts

@@ -199,6 +199,20 @@ export class DefaultProviderDatabase implements ProviderDatabase {
     }
   }
 
+  async listEntityProviderNames(txOpaque: Transaction): Promise<string[]> {
+    const tx = txOpaque as Knex | Knex.Transaction;
+
+    const rows = await tx<DbRefreshStateReferencesRow>(
+      'refresh_state_references',
+    )
+      .distinct('source_key')
+      .whereNotNull('source_key');
+
+    return rows
+      .map(row => row.source_key)
+      .filter((key): key is string => !!key);
+  }
+
   async refreshByRefreshKeys(
     txOpaque: Transaction,
     options: RefreshByKeyOptions,

plugins/catalog-backend/src/database/types.ts

@@ -173,6 +173,11 @@ export interface ProviderDatabase {
     txOpaque: Transaction,
     options: RefreshByKeyOptions,
   ): Promise<void>;
+
+  /**
+   * List the names of all the entity providers that have references in the provider database.
+   */
+  listEntityProviderNames(txOpaque: Transaction): Promise<string[]>;
 }
 
 // TODO(Rugvip): This is only partial for now

plugins/catalog-backend/src/processing/evictOrphanedEntityProviders.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2025 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { EntityProvider } from '@backstage/plugin-catalog-node';
+import { ProviderDatabase } from '../database/types';
+import { LoggerService } from '@backstage/backend-plugin-api';
+
+async function getOrphanedEntityProviderNames(
+  db: ProviderDatabase,
+  providers: EntityProvider[],
+): Promise<string[]> {
+  const dbProviderNames = await db.transaction(async tx =>
+    db.listEntityProviderNames(tx),
+  );
+
+  const providerNames = providers.map(p => p.getProviderName());
+
+  return dbProviderNames.filter(
+    dbProviderName => !providerNames.includes(dbProviderName),
+  );
+}
+
+async function removeEntitiesForProvider(
+  db: ProviderDatabase,
+  providerName: string,
+  logger: LoggerService,
+) {
+  try {
+    await db.transaction(async tx => {
+      await db.replaceUnprocessedEntities(tx, {
+        sourceKey: providerName,
+        type: 'full',
+        items: [],
+      });
+    });
+    logger.info(`Removed entities for orphaned provider ${providerName}`);
+  } catch (e) {
+    logger.error(
+      `Failed to remove entities for orphaned provider ${providerName}`,
+      e,
+    );
+  }
+}
+
+export async function evictOrphanedEntityProviders(
+  db: ProviderDatabase,
+  providers: EntityProvider[],
+  logger: LoggerService,
+) {
+  for (const providerName of await getOrphanedEntityProviderNames(
+    db,
+    providers,
+  )) {
+    await removeEntitiesForProvider(db, providerName, logger);
+  }
+}

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -87,6 +87,7 @@ import {
   ProcessingIntervalFunction,
 } from '../processing';
 import { connectEntityProviders } from '../processing/connectEntityProviders';
+import { evictOrphanedEntityProviders } from '../processing/evictOrphanedEntityProviders';
 import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine';
 import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator';
 import {
@@ -645,6 +646,14 @@ export class CatalogBuilder {
 
     await connectEntityProviders(providerDatabase, entityProviders);
 
+    if (config.getOptionalBoolean('catalog.evictOrphanedEntityProviders')) {
+      await evictOrphanedEntityProviders(
+        providerDatabase,
+        entityProviders,
+        logger,
+      );
+    }
+
     return {
       processingEngine: {
         async start() {