paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 32 Packages Projects Releases Wiki Activity

Catalog: Fix entity orphaning

Browse Source 
Co-authored-by: Patrik Oldsberg <poldsberg@gmail.com>
Signed-off-by: Johan Haals <johan.haals@gmail.com>
This commit is contained in:
Johan Haals
2021-10-13 11:30:56 +02:00
parent 8c754968d9
commit 3b59bb915e
6 changed files with 180 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/moody-shrimps-whisper.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend': patch
---
Fixes a bug in the catalog where entities were not being marked as orphaned.
plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts
+130
View File
@@ -1182,4 +1182,134 @@ describe('Default Processing Database', () => {
60_000,
);
});
describe('listAncestors', () => {
let nextId = 1;
function makeEntity(ref: string) {
return {
entity_id: String(nextId++),
entity_ref: ref,
unprocessed_entity: JSON.stringify({
kind: 'Location',
apiVersion: '1.0.0',
metadata: {
name: 'xyz',
},
}),
errors: '[]',
next_update_at: '2019-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
};
}
it.each(databases.eachSupportedId())(
'should return entities to process, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
await knex<DbRefreshStateRow>('refresh_state').insert(
makeEntity('location:default/root-1'),
);
await knex<DbRefreshStateRow>('refresh_state').insert(
makeEntity('location:default/root-2'),
);
await knex<DbRefreshStateRow>('refresh_state').insert(
makeEntity('component:default/foobar'),
);
await insertRefRow(knex, {
source_key: 'source',
target_entity_ref: 'location:default/root-2',
});
await insertRefRow(knex, {
source_entity_ref: 'location:default/root-2',
target_entity_ref: 'location:default/root-1',
});
await insertRefRow(knex, {
source_entity_ref: 'location:default/root-1',
target_entity_ref: 'component:default/foobar',
});
const result = await db.transaction(async tx =>
db.listAncestors(tx, { entityRef: 'component:default/foobar' }),
);
expect(result.entityRefs).toEqual([
'location:default/root-1',
'location:default/root-2',
]);
},
);
});
describe('listParents', () => {
let nextId = 1;
function makeEntity(ref: string) {
return {
entity_id: String(nextId++),
entity_ref: ref,
unprocessed_entity: JSON.stringify({
kind: 'Location',
apiVersion: '1.0.0',
metadata: {
name: 'xyz',
},
}),
errors: '[]',
next_update_at: '2019-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
};
}
it.each(databases.eachSupportedId())(
'should return entities to process, %p',
async databaseId => {
const { knex, db } = await createDatabase(databaseId);
await knex<DbRefreshStateRow>('refresh_state').insert(
makeEntity('location:default/root-1'),
);
await knex<DbRefreshStateRow>('refresh_state').insert(
makeEntity('location:default/root-2'),
);
await knex<DbRefreshStateRow>('refresh_state').insert(
makeEntity('component:default/foobar'),
);
await insertRefRow(knex, {
source_key: 'source',
target_entity_ref: 'location:default/root-2',
});
await insertRefRow(knex, {
source_entity_ref: 'location:default/root-2',
target_entity_ref: 'location:default/root-1',
});
await insertRefRow(knex, {
source_entity_ref: 'location:default/root-1',
target_entity_ref: 'component:default/foobar',
});
await insertRefRow(knex, {
source_entity_ref: 'location:default/root-2',
target_entity_ref: 'component:default/foobar',
});
const result1 = await db.transaction(async tx =>
db.listParents(tx, { entityRef: 'component:default/foobar' }),
);
expect(result1.entityRefs).toEqual([
'location:default/root-1',
'location:default/root-2',
]);
const result2 = await db.transaction(async tx =>
db.listParents(tx, { entityRef: 'location:default/root-1' }),
);
expect(result2.entityRefs).toEqual(['location:default/root-2']);
const result3 = await db.transaction(async tx =>
db.listParents(tx, { entityRef: 'location:default/root-2' }),
);
expect(result3.entityRefs).toEqual([]);
},
);
});
});
plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts
+19
View File
@@ -31,6 +31,8 @@ import {
ListAncestorsOptions,
ListAncestorsResult,
UpdateEntityCacheOptions,
ListParentsOptions,
ListParentsResult,
} from './types';
import { DeferredEntity } from '../processing/types';
import { RefreshIntervalFunction } from '../processing/refresh';
@@ -419,6 +421,23 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
);
}
async listParents(
txOpaque: Transaction,
options: ListParentsOptions,
): Promise<ListParentsResult> {
const tx = txOpaque as Knex.Transaction;
const rows = await tx<DbRefreshStateReferencesRow>(
'refresh_state_references',
)
.where({ target_entity_ref: options.entityRef })
.select();
const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);
return { entityRefs };
}
async refresh(txOpaque: Transaction, options: RefreshOptions): Promise<void> {
const tx = txOpaque as Knex.Transaction;
const { entityRef } = options;
plugins/catalog-backend/src/database/types.ts
+13
View File
@@ -91,6 +91,14 @@ export type ListAncestorsResult = {
entityRefs: string[];
};
export type ListParentsOptions = {
entityRef: string;
};
export type ListParentsResult = {
entityRefs: string[];
};
export interface ProcessingDatabase {
transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T>;
@@ -148,4 +156,9 @@ export interface ProcessingDatabase {
txOpaque: Transaction,
options: ListAncestorsOptions,
): Promise<ListAncestorsResult>;
listParents(
txOpaque: Transaction,
options: ListParentsOptions,
): Promise<ListParentsResult>;
}
plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts
+4 -1
View File
@@ -29,6 +29,7 @@ describe('DefaultCatalogProcessingEngine', () => {
getProcessableEntities: jest.fn(),
updateProcessedEntity: jest.fn(),
updateEntityCache: jest.fn(),
listParents: jest.fn(),
} as unknown as jest.Mocked<DefaultProcessingDatabase>;
const orchestrator: jest.Mocked<CatalogProcessingOrchestrator> = {
process: jest.fn(),
@@ -209,18 +210,19 @@ describe('DefaultCatalogProcessingEngine', () => {
db.transaction.mockImplementation(cb => cb((() => {}) as any));
db.listParents.mockResolvedValue({ entityRefs: [] });
db.getProcessableEntities
.mockResolvedValueOnce({
items: [{ ...refreshState, resultHash: 'NOT RIGHT' }],
})
.mockResolvedValue({ items: [] });
await engine.start();
await waitForExpect(() => {
expect(orchestrator.process).toBeCalledTimes(1);
expect(hash.digest).toBeCalledTimes(1);
expect(db.updateProcessedEntity).toBeCalledTimes(1);
expect(db.listParents).toBeCalledTimes(1);
});
expect(db.updateEntityCache).not.toHaveBeenCalled();
@@ -236,6 +238,7 @@ describe('DefaultCatalogProcessingEngine', () => {
expect(hash.digest).toBeCalledTimes(2);
expect(db.updateProcessedEntity).toBeCalledTimes(1);
expect(db.updateEntityCache).toBeCalledTimes(1);
expect(db.listParents).toBeCalledTimes(2);
});
expect(db.updateEntityCache).toHaveBeenCalledWith(expect.anything(), {
id: '',
plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts
+9 -1
View File
@@ -189,10 +189,18 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine {
let hashBuilder = this.createHash().update(errorsString);
if (result.ok) {
const { entityRefs: parents } =
await this.processingDatabase.transaction(tx =>
this.processingDatabase.listParents(tx, {
entityRef,
}),
);
hashBuilder = hashBuilder
.update(stableStringify({ ...result.completedEntity }))
.update(stableStringify([...result.deferredEntities]))
.update(stableStringify([...result.relations]));
.update(stableStringify([...result.relations]))
.update(stableStringify([...parents]));
}
const resultHash = hashBuilder.digest('hex');