paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 36 Packages Projects Releases Wiki Activity

Merge pull request #28971 from UsainBloot/catalog/evict-orphaned-entity-provider

Browse Source 
Feature: [Catalog] Remove entities from old EntityProviders by default
This commit is contained in:
Patrik Oldsberg
2025-02-27 11:48:57 +01:00
committed by GitHub
parent 5076f4999a abd0ab7901
commit a852584066
9 changed files with 299 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/itchy-moose-raise.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend': minor
---
Added opt-in ability to evict entities from the catalog whose provider is no longer configured. See [Catalog configuration documentation](https://backstage.io/docs/features/software-catalog/configuration#clean-up-entities-from-orphaned-entity-providers)
docs/features/software-catalog/configuration.md
+11
View File
@@ -163,6 +163,17 @@ catalog:
orphanStrategy: delete
```
## Clean up entities from orphaned entity providers
By default, if an entity provider which has provided entities to the catalog, is no longer configured, then the entities remain in the catalog until they are manually unregistered.
To remove these entities automatically, you can use the following configuration.
```yaml
catalog:
orphanProviderStrategy: delete
```
## Processing Interval
The [processing loop](./life-of-an-entity.md#processing) is
plugins/catalog-backend/config.d.ts
Vendored
+7
View File
@@ -155,6 +155,13 @@ export interface Config {
*/
orphanStrategy?: 'keep' | 'delete';
/**
* The strategy to use for entities that are referenced by providers that are orphaned,
* i.e. entities with no providers currently configured in the catalog. The default value is
* "keep".
*/
orphanProviderStrategy?: 'keep' | 'delete';
/**
* The strategy to use when stitching together the final entities.
*/
plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts
+96 -14
View File
@@ -59,21 +59,21 @@ describe('DefaultProviderDatabase', () => {
await db<DbRefreshStateRow>('refresh_state').insert(ref);
};
describe('replaceUnprocessedEntities', () => {
const createLocations = async (db: Knex, entityRefs: string[]) => {
for (const ref of entityRefs) {
await insertRefreshStateRow(db, {
entity_id: uuid.v4(),
entity_ref: ref,
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
});
}
};
const createLocations = async (db: Knex, entityRefs: string[]) => {
for (const ref of entityRefs) {
await insertRefreshStateRow(db, {
entity_id: uuid.v4(),
entity_ref: ref,
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
});
}
};
describe('replaceUnprocessedEntities', () => {
it.each(databases.eachSupportedId())(
'replaces all existing state correctly for simple dependency chains, %p',
async databaseId => {
@@ -988,4 +988,86 @@ describe('DefaultProviderDatabase', () => {
},
);
});
describe('listReferenceSourceKeys', () => {
it.each(databases.eachSupportedId())(
'returns the source_keys from "refresh_state_references", %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
await createLocations(knex, [
'location:default/root',
'location:default/root-1',
]);
await insertRefRow(knex, {
source_key: 'foo',
target_entity_ref: 'location:default/root',
});
await insertRefRow(knex, {
source_key: 'bar',
target_entity_ref: 'location:default/root-1',
});
const res = await db.transaction(async tx =>
db.listReferenceSourceKeys(tx),
);
expect(res).toEqual(['bar', 'foo']);
},
);
it.each(databases.eachSupportedId())(
'returns only unique source_keys", %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
await createLocations(knex, [
'location:default/root',
'location:default/root-1',
]);
await insertRefRow(knex, {
source_key: 'foo',
target_entity_ref: 'location:default/root',
});
await insertRefRow(knex, {
source_key: 'foo',
target_entity_ref: 'location:default/root-1',
});
const res = await db.transaction(async tx =>
db.listReferenceSourceKeys(tx),
);
expect(res).toEqual(['foo']);
},
);
it.each(databases.eachSupportedId())(
'does not return null source_keys", %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
await createLocations(knex, [
'location:default/root',
'location:default/root-1',
]);
await insertRefRow(knex, {
source_key: 'foo',
target_entity_ref: 'location:default/root',
});
await insertRefRow(knex, {
target_entity_ref: 'location:default/root-1',
});
const res = await db.transaction(async tx =>
db.listReferenceSourceKeys(tx),
);
expect(res).toEqual(['foo']);
},
);
});
});
plugins/catalog-backend/src/database/DefaultProviderDatabase.ts
+14
View File
@@ -199,6 +199,20 @@ export class DefaultProviderDatabase implements ProviderDatabase {
}
}
async listReferenceSourceKeys(txOpaque: Transaction): Promise<string[]> {
const tx = txOpaque as Knex | Knex.Transaction;
const rows = await tx<DbRefreshStateReferencesRow>(
'refresh_state_references',
)
.distinct('source_key')
.whereNotNull('source_key');
return rows
.map(row => row.source_key)
.filter((key): key is string => !!key);
}
async refreshByRefreshKeys(
txOpaque: Transaction,
options: RefreshByKeyOptions,
plugins/catalog-backend/src/database/types.ts
+5
View File
@@ -173,6 +173,11 @@ export interface ProviderDatabase {
txOpaque: Transaction,
options: RefreshByKeyOptions,
): Promise<void>;
/**
* List the names of all the entity providers that have references in the provider database.
*/
listReferenceSourceKeys(txOpaque: Transaction): Promise<string[]>;
}
// TODO(Rugvip): This is only partial for now
plugins/catalog-backend/src/processing/evictEntitiesFromOrphanedProviders.test.ts
+74
View File
@@ -0,0 +1,74 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityProvider } from '@backstage/plugin-catalog-node';
import { mockServices } from '@backstage/backend-test-utils';
import { DefaultProviderDatabase } from '../database/DefaultProviderDatabase';
import { evictEntitiesFromOrphanedProviders } from './evictEntitiesFromOrphanedProviders';
describe('evictEntitiesFromOrphanedProviders', () => {
const db = {
transaction: jest.fn().mockImplementation(cb => cb((() => {}) as any)),
replaceUnprocessedEntities: jest.fn(),
listReferenceSourceKeys: jest.fn(),
} as unknown as jest.Mocked<DefaultProviderDatabase>;
const providers = [
{ getProviderName: () => 'provider1' },
{ getProviderName: () => 'provider2' },
] as unknown as EntityProvider[];
const logger = mockServices.logger.mock();
it('replaces unprocessed entities for orphaned providers with empty items', async () => {
db.listReferenceSourceKeys.mockResolvedValue(['foo', 'bar']);
await evictEntitiesFromOrphanedProviders({ db, providers, logger });
expect(db.replaceUnprocessedEntities).toHaveBeenCalledTimes(2);
expect(db.replaceUnprocessedEntities).toHaveBeenNthCalledWith(
1,
expect.anything(),
{
sourceKey: 'foo',
type: 'full',
items: [],
},
);
expect(db.replaceUnprocessedEntities).toHaveBeenNthCalledWith(
2,
expect.anything(),
{
sourceKey: 'bar',
type: 'full',
items: [],
},
);
});
it('does not replace unprocessed entities for providers that are not orphaned', async () => {
db.listReferenceSourceKeys.mockResolvedValue(['foo', 'provider1']);
await evictEntitiesFromOrphanedProviders({ db, providers, logger });
expect(db.replaceUnprocessedEntities).not.toHaveBeenCalledWith(
expect.anything(),
{
sourceKey: 'provider1',
type: 'full',
items: [],
},
);
});
});
plugins/catalog-backend/src/processing/evictEntitiesFromOrphanedProviders.ts
+77
View File
@@ -0,0 +1,77 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityProvider } from '@backstage/plugin-catalog-node';
import { LoggerService } from '@backstage/backend-plugin-api';
import { ProviderDatabase } from '../database/types';
async function getOrphanedEntityProviderNames({
db,
providers,
}: {
db: ProviderDatabase;
providers: EntityProvider[];
}): Promise<string[]> {
const dbProviderNames = await db.transaction(async tx =>
db.listReferenceSourceKeys(tx),
);
const providerNames = providers.map(p => p.getProviderName());
return dbProviderNames.filter(
dbProviderName => !providerNames.includes(dbProviderName),
);
}
async function removeEntitiesForProvider({
db,
providerName,
logger,
}: {
db: ProviderDatabase;
providerName: string;
logger: LoggerService;
}) {
try {
await db.transaction(async tx => {
await db.replaceUnprocessedEntities(tx, {
sourceKey: providerName,
type: 'full',
items: [],
});
});
logger.info(`Removed entities for orphaned provider ${providerName}`);
} catch (e) {
logger.error(
`Failed to remove entities for orphaned provider ${providerName}`,
e,
);
}
}
export async function evictEntitiesFromOrphanedProviders(options: {
db: ProviderDatabase;
providers: EntityProvider[];
logger: LoggerService;
}) {
for (const providerName of await getOrphanedEntityProviderNames(options)) {
await removeEntitiesForProvider({
db: options.db,
providerName,
logger: options.logger,
});
}
}
plugins/catalog-backend/src/service/CatalogBuilder.ts
+10
View File
@@ -87,6 +87,7 @@ import {
ProcessingIntervalFunction,
} from '../processing';
import { connectEntityProviders } from '../processing/connectEntityProviders';
import { evictEntitiesFromOrphanedProviders } from '../processing/evictEntitiesFromOrphanedProviders';
import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine';
import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator';
import {
@@ -643,6 +644,15 @@ export class CatalogBuilder {
disableRelationsCompatibility,
});
if (
config.getOptionalString('catalog.orphanProviderStrategy') === 'delete'
) {
await evictEntitiesFromOrphanedProviders({
db: providerDatabase,
providers: entityProviders,
logger,
});
}
await connectEntityProviders(providerDatabase, entityProviders);
return {