paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 37 Packages Projects Releases Wiki Activity

simplify

Browse Source 
Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
2023-02-12 12:13:16 +01:00
parent 1aec041c34
commit 33d042bcaf
1 changed files with 145 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.ts
+145 -128
View File
@@ -16,7 +16,7 @@
import { Knex } from 'knex';
import lodash from 'lodash';
import { DbRefreshStateReferencesRow, DbRefreshStateRow } from '../../tables';
import { DbRefreshStateReferencesRow } from '../../tables';
/**
* Given a number of entity refs originally created by a given entity provider
@@ -36,135 +36,152 @@ export async function deleteWithEagerPruningOfChildren(options: {
// limits for the number of permitted bindings on a precompiled statement
let removedCount = 0;
for (const refs of lodash.chunk(entityRefs, 1000)) {
/*
WITH RECURSIVE
-- All the nodes that can be reached downwards from our root
descendants(entity_ref) AS (
SELECT target_entity_ref
FROM refresh_state_references
WHERE source_key = "R1" AND target_entity_ref IN [...refs]
UNION
SELECT target_entity_ref
FROM descendants
JOIN refresh_state_references ON source_entity_ref = descendants.entity_ref
),
-- All the individual relations that can be reached upwards from each descendant
ancestors(source_key, source_entity_ref, target_entity_ref, subject) AS (
SELECT source_key, source_entity_ref, target_entity_ref, descendants.entity_ref
FROM descendants
JOIN refresh_state_references ON refresh_state_references.target_entity_ref = descendants.entity_ref
UNION
SELECT
refresh_state_references.source_key,
refresh_state_references.source_entity_ref,
refresh_state_references.target_entity_ref,
ancestors.subject
FROM ancestors
JOIN refresh_state_references ON refresh_state_references.target_entity_ref = ancestors.source_entity_ref
)
-- Start out with all of the descendants
SELECT descendants.entity_ref
FROM descendants
-- Exclude those who seem to have a root relation somewhere upwards that's not part of our own deletion
WHERE NOT EXISTS (
SELECT * FROM ancestors
WHERE ancestors.subject = descendants.entity_ref
AND ancestors.source_key IS NOT NULL
AND (ancestors.source_key != "R1" OR ancestors.target_entity_ref NOT IN [...refs])
)
*/
removedCount += await tx<DbRefreshStateRow>('refresh_state')
.whereIn('entity_ref', function orphanedEntityRefs(orphans) {
return (
orphans
// All the nodes that can be reached downwards from our root
.withRecursive(
'descendants',
['entity_ref'],
function descendants(outer) {
return outer
.select({ entity_ref: 'target_entity_ref' })
.from('refresh_state_references')
.where('source_key', '=', sourceKey)
.whereIn('target_entity_ref', refs)
.union(function recursive(inner) {
return inner
.select({
entity_ref:
'refresh_state_references.target_entity_ref',
})
.from('descendants')
.join('refresh_state_references', {
'descendants.entity_ref':
'refresh_state_references.source_entity_ref',
});
});
},
)
// All the relations that can be reached upwards from each descendant
.withRecursive(
'ancestors',
[
'source_key',
'source_entity_ref',
'target_entity_ref',
'subject',
],
function ancestors(outer) {
return outer
.select({
source_key: 'refresh_state_references.source_key',
source_entity_ref:
'refresh_state_references.source_entity_ref',
target_entity_ref:
'refresh_state_references.target_entity_ref',
subject: 'descendants.entity_ref',
})
removedCount += await tx
.delete()
.from('refresh_state')
.whereIn('entity_ref', orphans =>
orphans
// First find all nodes that can be reached downwards from the roots
// (deletion targets), including the roots themselves, by traversing
// down the refresh_state_references table. Note that this query
// starts with a condition that source_key = our source key, and
// target_entity_ref is one of the deletion targets. This has two
// effects: it won't match attempts at deleting something that didn't
// originate from us in the first place, and also won't match non-root
// entities (source_key would be null for those).
//
// KeyA - R1 - R2 Legend:
// \ -----------------------------------------
// R3 Key* Source key
// / R* Entity ref
// KeyA - R4 - R5 lines Individual references; sources to
// / the left and targets to the right
// KeyB --- R6
//
// The scenario is that KeyA wants to delete R1.
//
// The query starts with the KeyA-R1 reference, and then traverses
// down to also find R2 and R3. It uses union instead of union all,
// because it wants to find the set of unique descendants even if
// the tree has unexpected loops etc.
.withRecursive('descendants', ['entity_ref'], initial =>
initial
.select('target_entity_ref')
.from('refresh_state_references')
.where('source_key', '=', sourceKey)
.whereIn('target_entity_ref', refs)
.union(recursive =>
recursive
.select('refresh_state_references.target_entity_ref')
.from('descendants')
.join('refresh_state_references', {
'refresh_state_references.target_entity_ref':
'descendants.entity_ref',
})
.union(function recursive(inner) {
return inner
.select({
source_key: 'refresh_state_references.source_key',
source_entity_ref:
'refresh_state_references.source_entity_ref',
target_entity_ref:
'refresh_state_references.target_entity_ref',
subject: 'ancestors.subject',
})
.from('ancestors')
.join('refresh_state_references', {
'refresh_state_references.target_entity_ref':
'ancestors.source_entity_ref',
});
});
},
)
// Start out with all of the descendants
.select('descendants.entity_ref')
.from('descendants')
// Exclude those who seem to have a root relation somewhere upwards that's not part of our own deletion
.whereNotExists(function otherAncestors(outer) {
outer
.from('ancestors')
.where(
'ancestors.subject',
'=',
tx.ref('descendants.entity_ref'),
.join(
'refresh_state_references',
'descendants.entity_ref',
'refresh_state_references.source_entity_ref',
),
),
)
// Then for each descendant, traverse all the way back upwards through
// the refresh_state_references table to get an exhaustive list of all
// references that are part of keeping that particular descendant
// alive.
//
// Continuing the scenario from above, starting from R3, it goes
// upwards to find every pair along every relation line.
//
// Top branch: R2-R3, R1-R2, KeyA-R1
// Middle branch: R5-R3, R4-R5, KeyA-R4
// Bottom branch: R6-R5, KeyB-R6
//
// Note that this all applied to the subject R3. The exact same thing
// will be done starting from each other descendant (R2 and R1). They
// only have one and two references to find, respectively.
//
// This query also uses union instead of union all, to get the set of
// distinct relations even if the tree has unexpected loops etc.
.withRecursive(
'ancestors',
['source_key', 'source_entity_ref', 'target_entity_ref', 'subject'],
initial =>
initial
.select(
'refresh_state_references.source_key',
'refresh_state_references.source_entity_ref',
'refresh_state_references.target_entity_ref',
'descendants.entity_ref',
)
.whereNotNull('ancestors.source_key')
.andWhere(function differentRoot(inner) {
inner
.where('ancestors.source_key', '!=', sourceKey)
.orWhereNotIn('ancestors.target_entity_ref', refs);
});
})
);
})
.delete();
.from('descendants')
.join(
'refresh_state_references',
'refresh_state_references.target_entity_ref',
'descendants.entity_ref',
)
.union(recursive =>
recursive
.select(
'refresh_state_references.source_key',
'refresh_state_references.source_entity_ref',
'refresh_state_references.target_entity_ref',
'ancestors.subject',
)
.from('ancestors')
.join(
'refresh_state_references',
'refresh_state_references.target_entity_ref',
'ancestors.source_entity_ref',
),
),
)
// Finally, from that list of ancestor relations per descendant, pick
// out the ones that are roots (have a source_key). Specifically, find
// ones that seem to be be either (1) from another source, or (2)
// aren't part of the deletion targets. Those are markers that tell us
// that the corresponding descendant should be kept alive and NOT
// subject to eager deletion, because there's "something else" (not
// targeted for deletion) that has references down through the tree to
// it.
//
// Continuing the scenario from above, for R3 we have
//
// KeyA-R1, KeyA-R4, KeyB-R6
//
// This tells us that R3 should be kept alive for two reasons: it's
// referenced by a node that isn't being deleted (R4), and also by
// another source (KeyB). What about R1 and R2? They both have
//
// KeyA-R1
//
// So those should be deleted, since they are definitely only being
// kept alive by something that's about to be deleted.
//
// Final shape of the tree:
//
// R3
// /
// KeyA - R4 - R5
// /
// KeyB --- R6
.with('retained', ['entity_ref'], notPartOfDeletion =>
notPartOfDeletion
.select('subject')
.from('ancestors')
.whereNotNull('ancestors.source_key')
.where(foreignKeyOrRef =>
foreignKeyOrRef
.where('ancestors.source_key', '!=', sourceKey)
.orWhereNotIn('ancestors.target_entity_ref', refs),
),
)
// Return all descendants minus the retained ones
.select('descendants.entity_ref')
.from('descendants')
.leftOuterJoin(
'retained',
'retained.entity_ref',
'descendants.entity_ref',
)
.whereNull('retained.entity_ref'),
);
// Delete the references that originate only from this entity provider. Note
// that there may be more than one entity provider making a "claim" for a