From 87082cf42ec64ab3778d3be7b0941713cb0fe1be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 13 Apr 2023 23:30:38 +0200 Subject: [PATCH] fix: use relations instead of references to deduce what to reprocess MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../util/deleteOrphanedEntities.test.ts | 48 +++++++++++++------ .../operations/util/deleteOrphanedEntities.ts | 14 +++--- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/plugins/catalog-backend/src/database/operations/util/deleteOrphanedEntities.test.ts b/plugins/catalog-backend/src/database/operations/util/deleteOrphanedEntities.test.ts index 9d1f991590..243b5db1b9 100644 --- a/plugins/catalog-backend/src/database/operations/util/deleteOrphanedEntities.test.ts +++ b/plugins/catalog-backend/src/database/operations/util/deleteOrphanedEntities.test.ts @@ -22,6 +22,7 @@ import { DbFinalEntitiesRow, DbRefreshStateReferencesRow, DbRefreshStateRow, + DbRelationsRow, } from '../../tables'; import { deleteOrphanedEntities } from './deleteOrphanedEntities'; @@ -55,15 +56,6 @@ describe('deleteOrphanedEntities', () => { return result!; } - async function insertReference( - knex: Knex, - ...refs: DbRefreshStateReferencesRow[] - ) { - return knex('refresh_state_references').insert( - refs, - ); - } - async function insertEntity(knex: Knex, ...entityRefs: string[]) { for (const ref of entityRefs) { const entityId = uuid.v4(); @@ -85,6 +77,28 @@ describe('deleteOrphanedEntities', () => { } } + async function insertReference( + knex: Knex, + ...refs: DbRefreshStateReferencesRow[] + ) { + await knex('refresh_state_references').insert( + refs, + ); + } + + async function insertRelation(knex: Knex, fromRef: string, toRef: string) { + const orig = await knex + .select('entity_id') + .from('refresh_state') + .where('entity_ref', fromRef); + await knex('relations').insert({ + originating_entity_id: orig[0].entity_id, + type: 'fake', + source_entity_ref: fromRef, + target_entity_ref: toRef, + }); + } + async function refreshState(knex: Knex) { return await knex('refresh_state') .orderBy('entity_ref') @@ -109,6 +123,8 @@ describe('deleteOrphanedEntities', () => { 'works for some mixed paths, %p', async databaseId => { /* + In this graph, edges represent refresh state references, not entity relations: + P1 - E1 -- E2 / E3 @@ -127,7 +143,8 @@ describe('deleteOrphanedEntities', () => { E10 - Result: E3, E4, E5, E6, and E10 deleted; others remain; children of deleted orphans marked for reprocessing + Result: E3, E4, E5, E6, and E10 deleted; others remain + Entities that had relations pointing at orphans are marked for reprocessing */ const knex = await createDatabase(databaseId); await insertEntity( @@ -156,18 +173,21 @@ describe('deleteOrphanedEntities', () => { { source_entity_ref: 'E8', target_entity_ref: 'E7' }, { source_key: 'P3', target_entity_ref: 'E9' }, ); + await insertRelation(knex, 'E1', 'E2'); + await insertRelation(knex, 'E2', 'E3'); + await insertRelation(knex, 'E10', 'E6'); await expect(run(knex)).resolves.toEqual(5); await expect(refreshState(knex)).resolves.toEqual([ { entity_ref: 'E1', result_hash: 'original' }, - { entity_ref: 'E2', result_hash: 'orphan-parent-deleted' }, - { entity_ref: 'E7', result_hash: 'orphan-parent-deleted' }, + { entity_ref: 'E2', result_hash: 'orphan-relation-deleted' }, + { entity_ref: 'E7', result_hash: 'original' }, { entity_ref: 'E8', result_hash: 'original' }, { entity_ref: 'E9', result_hash: 'original' }, ]); await expect(finalEntities(knex)).resolves.toEqual([ { entity_ref: 'E1', hash: 'original' }, - { entity_ref: 'E2', hash: 'orphan-parent-deleted' }, - { entity_ref: 'E7', hash: 'orphan-parent-deleted' }, + { entity_ref: 'E2', hash: 'orphan-relation-deleted' }, + { entity_ref: 'E7', hash: 'original' }, { entity_ref: 'E8', hash: 'original' }, { entity_ref: 'E9', hash: 'original' }, ]); diff --git a/plugins/catalog-backend/src/database/operations/util/deleteOrphanedEntities.ts b/plugins/catalog-backend/src/database/operations/util/deleteOrphanedEntities.ts index ab31440b19..920acae7bd 100644 --- a/plugins/catalog-backend/src/database/operations/util/deleteOrphanedEntities.ts +++ b/plugins/catalog-backend/src/database/operations/util/deleteOrphanedEntities.ts @@ -43,18 +43,18 @@ export async function deleteOrphanedEntities(options: { ) .select({ entityId: 'orphans.entity_id', - relationTargetId: 'refresh_state.entity_id', + relationSourceId: 'refresh_state.entity_id', }) .from('orphans') .leftOuterJoin( - 'refresh_state_references', - 'refresh_state_references.source_entity_ref', + 'relations', + 'relations.target_entity_ref', 'orphans.entity_ref', ) .leftOuterJoin( 'refresh_state', 'refresh_state.entity_ref', - 'refresh_state_references.target_entity_ref', + 'relations.source_entity_ref', ); if (!candidates.length) { @@ -63,7 +63,7 @@ export async function deleteOrphanedEntities(options: { const orphanIds: string[] = uniq(candidates.map(r => r.entityId)); const orphanRelationIds: string[] = uniq( - candidates.map(r => r.relationTargetId).filter(Boolean), + candidates.map(r => r.relationSourceId).filter(Boolean), ); total += orphanIds.length; @@ -79,13 +79,13 @@ export async function deleteOrphanedEntities(options: { await tx .table('final_entities') .update({ - hash: 'orphan-parent-deleted', + hash: 'orphan-relation-deleted', }) .whereIn('entity_id', orphanRelationIds); await tx .table('refresh_state') .update({ - result_hash: 'orphan-parent-deleted', + result_hash: 'orphan-relation-deleted', next_update_at: tx.fn.now(), }) .whereIn('entity_id', orphanRelationIds);