From 33d042bcaf0b6dc4c045ab63e655c51203871170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sun, 12 Feb 2023 12:13:16 +0100 Subject: [PATCH] simplify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../deleteWithEagerPruningOfChildren.ts | 273 ++++++++++-------- 1 file changed, 145 insertions(+), 128 deletions(-) diff --git a/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.ts b/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.ts index c600825d6f..33da37bc55 100644 --- a/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.ts +++ b/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.ts @@ -16,7 +16,7 @@ import { Knex } from 'knex'; import lodash from 'lodash'; -import { DbRefreshStateReferencesRow, DbRefreshStateRow } from '../../tables'; +import { DbRefreshStateReferencesRow } from '../../tables'; /** * Given a number of entity refs originally created by a given entity provider @@ -36,135 +36,152 @@ export async function deleteWithEagerPruningOfChildren(options: { // limits for the number of permitted bindings on a precompiled statement let removedCount = 0; for (const refs of lodash.chunk(entityRefs, 1000)) { - /* - WITH RECURSIVE - -- All the nodes that can be reached downwards from our root - descendants(entity_ref) AS ( - SELECT target_entity_ref - FROM refresh_state_references - WHERE source_key = "R1" AND target_entity_ref IN [...refs] - UNION - SELECT target_entity_ref - FROM descendants - JOIN refresh_state_references ON source_entity_ref = descendants.entity_ref - ), - -- All the individual relations that can be reached upwards from each descendant - ancestors(source_key, source_entity_ref, target_entity_ref, subject) AS ( - SELECT source_key, source_entity_ref, target_entity_ref, descendants.entity_ref - FROM descendants - JOIN refresh_state_references ON refresh_state_references.target_entity_ref = descendants.entity_ref - UNION - SELECT - refresh_state_references.source_key, - refresh_state_references.source_entity_ref, - refresh_state_references.target_entity_ref, - ancestors.subject - FROM ancestors - JOIN refresh_state_references ON refresh_state_references.target_entity_ref = ancestors.source_entity_ref - ) - -- Start out with all of the descendants - SELECT descendants.entity_ref - FROM descendants - -- Exclude those who seem to have a root relation somewhere upwards that's not part of our own deletion - WHERE NOT EXISTS ( - SELECT * FROM ancestors - WHERE ancestors.subject = descendants.entity_ref - AND ancestors.source_key IS NOT NULL - AND (ancestors.source_key != "R1" OR ancestors.target_entity_ref NOT IN [...refs]) - ) - */ - removedCount += await tx('refresh_state') - .whereIn('entity_ref', function orphanedEntityRefs(orphans) { - return ( - orphans - // All the nodes that can be reached downwards from our root - .withRecursive( - 'descendants', - ['entity_ref'], - function descendants(outer) { - return outer - .select({ entity_ref: 'target_entity_ref' }) - .from('refresh_state_references') - .where('source_key', '=', sourceKey) - .whereIn('target_entity_ref', refs) - .union(function recursive(inner) { - return inner - .select({ - entity_ref: - 'refresh_state_references.target_entity_ref', - }) - .from('descendants') - .join('refresh_state_references', { - 'descendants.entity_ref': - 'refresh_state_references.source_entity_ref', - }); - }); - }, - ) - // All the relations that can be reached upwards from each descendant - .withRecursive( - 'ancestors', - [ - 'source_key', - 'source_entity_ref', - 'target_entity_ref', - 'subject', - ], - function ancestors(outer) { - return outer - .select({ - source_key: 'refresh_state_references.source_key', - source_entity_ref: - 'refresh_state_references.source_entity_ref', - target_entity_ref: - 'refresh_state_references.target_entity_ref', - subject: 'descendants.entity_ref', - }) + removedCount += await tx + .delete() + .from('refresh_state') + .whereIn('entity_ref', orphans => + orphans + // First find all nodes that can be reached downwards from the roots + // (deletion targets), including the roots themselves, by traversing + // down the refresh_state_references table. Note that this query + // starts with a condition that source_key = our source key, and + // target_entity_ref is one of the deletion targets. This has two + // effects: it won't match attempts at deleting something that didn't + // originate from us in the first place, and also won't match non-root + // entities (source_key would be null for those). + // + // KeyA - R1 - R2 Legend: + // \ ----------------------------------------- + // R3 Key* Source key + // / R* Entity ref + // KeyA - R4 - R5 lines Individual references; sources to + // / the left and targets to the right + // KeyB --- R6 + // + // The scenario is that KeyA wants to delete R1. + // + // The query starts with the KeyA-R1 reference, and then traverses + // down to also find R2 and R3. It uses union instead of union all, + // because it wants to find the set of unique descendants even if + // the tree has unexpected loops etc. + .withRecursive('descendants', ['entity_ref'], initial => + initial + .select('target_entity_ref') + .from('refresh_state_references') + .where('source_key', '=', sourceKey) + .whereIn('target_entity_ref', refs) + .union(recursive => + recursive + .select('refresh_state_references.target_entity_ref') .from('descendants') - .join('refresh_state_references', { - 'refresh_state_references.target_entity_ref': - 'descendants.entity_ref', - }) - .union(function recursive(inner) { - return inner - .select({ - source_key: 'refresh_state_references.source_key', - source_entity_ref: - 'refresh_state_references.source_entity_ref', - target_entity_ref: - 'refresh_state_references.target_entity_ref', - subject: 'ancestors.subject', - }) - .from('ancestors') - .join('refresh_state_references', { - 'refresh_state_references.target_entity_ref': - 'ancestors.source_entity_ref', - }); - }); - }, - ) - // Start out with all of the descendants - .select('descendants.entity_ref') - .from('descendants') - // Exclude those who seem to have a root relation somewhere upwards that's not part of our own deletion - .whereNotExists(function otherAncestors(outer) { - outer - .from('ancestors') - .where( - 'ancestors.subject', - '=', - tx.ref('descendants.entity_ref'), + .join( + 'refresh_state_references', + 'descendants.entity_ref', + 'refresh_state_references.source_entity_ref', + ), + ), + ) + // Then for each descendant, traverse all the way back upwards through + // the refresh_state_references table to get an exhaustive list of all + // references that are part of keeping that particular descendant + // alive. + // + // Continuing the scenario from above, starting from R3, it goes + // upwards to find every pair along every relation line. + // + // Top branch: R2-R3, R1-R2, KeyA-R1 + // Middle branch: R5-R3, R4-R5, KeyA-R4 + // Bottom branch: R6-R5, KeyB-R6 + // + // Note that this all applied to the subject R3. The exact same thing + // will be done starting from each other descendant (R2 and R1). They + // only have one and two references to find, respectively. + // + // This query also uses union instead of union all, to get the set of + // distinct relations even if the tree has unexpected loops etc. + .withRecursive( + 'ancestors', + ['source_key', 'source_entity_ref', 'target_entity_ref', 'subject'], + initial => + initial + .select( + 'refresh_state_references.source_key', + 'refresh_state_references.source_entity_ref', + 'refresh_state_references.target_entity_ref', + 'descendants.entity_ref', ) - .whereNotNull('ancestors.source_key') - .andWhere(function differentRoot(inner) { - inner - .where('ancestors.source_key', '!=', sourceKey) - .orWhereNotIn('ancestors.target_entity_ref', refs); - }); - }) - ); - }) - .delete(); + .from('descendants') + .join( + 'refresh_state_references', + 'refresh_state_references.target_entity_ref', + 'descendants.entity_ref', + ) + .union(recursive => + recursive + .select( + 'refresh_state_references.source_key', + 'refresh_state_references.source_entity_ref', + 'refresh_state_references.target_entity_ref', + 'ancestors.subject', + ) + .from('ancestors') + .join( + 'refresh_state_references', + 'refresh_state_references.target_entity_ref', + 'ancestors.source_entity_ref', + ), + ), + ) + // Finally, from that list of ancestor relations per descendant, pick + // out the ones that are roots (have a source_key). Specifically, find + // ones that seem to be be either (1) from another source, or (2) + // aren't part of the deletion targets. Those are markers that tell us + // that the corresponding descendant should be kept alive and NOT + // subject to eager deletion, because there's "something else" (not + // targeted for deletion) that has references down through the tree to + // it. + // + // Continuing the scenario from above, for R3 we have + // + // KeyA-R1, KeyA-R4, KeyB-R6 + // + // This tells us that R3 should be kept alive for two reasons: it's + // referenced by a node that isn't being deleted (R4), and also by + // another source (KeyB). What about R1 and R2? They both have + // + // KeyA-R1 + // + // So those should be deleted, since they are definitely only being + // kept alive by something that's about to be deleted. + // + // Final shape of the tree: + // + // R3 + // / + // KeyA - R4 - R5 + // / + // KeyB --- R6 + .with('retained', ['entity_ref'], notPartOfDeletion => + notPartOfDeletion + .select('subject') + .from('ancestors') + .whereNotNull('ancestors.source_key') + .where(foreignKeyOrRef => + foreignKeyOrRef + .where('ancestors.source_key', '!=', sourceKey) + .orWhereNotIn('ancestors.target_entity_ref', refs), + ), + ) + // Return all descendants minus the retained ones + .select('descendants.entity_ref') + .from('descendants') + .leftOuterJoin( + 'retained', + 'retained.entity_ref', + 'descendants.entity_ref', + ) + .whereNull('retained.entity_ref'), + ); // Delete the references that originate only from this entity provider. Note // that there may be more than one entity provider making a "claim" for a