paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 24 Packages Projects Releases Wiki Activity

Merge pull request #15767 from RoadieHQ/improve-fact-lifecycle-performance

Browse Source 
Improve fact lifecycle performance
This commit is contained in:
Fredrik Adelöw
2023-01-16 14:07:52 +01:00
committed by GitHub
parent 58d2068149 3262f0ec83
commit 2c4b116ccc
2 changed files with 51 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts
+45 -43
View File
@@ -119,10 +119,10 @@ export class TechInsightsDatabase implements TechInsightsStore {
if (lifecycle && isTtl(lifecycle)) {
const expiration = DateTime.now().minus(lifecycle.timeToLive);
await this.deleteExpiredFactsByDate(tx, factRows, expiration);
await this.deleteExpiredFactsByDate(tx, id, expiration);
}
if (lifecycle && isMaxItems(lifecycle)) {
await this.deleteExpiredFactsByNumber(tx, factRows, lifecycle.maxItems);
await this.deleteExpiredFactsByNumber(tx, id, lifecycle.maxItems);
}
});
}
@@ -200,13 +200,13 @@ export class TechInsightsDatabase implements TechInsightsStore {
private async deleteExpiredFactsByDate(
tx: Transaction,
factRows: { id: string; entity: string }[],
factRetrieverId: string,
timestamp: DateTime,
) {
await tx<RawDbFactRow>('facts')
.whereIn(
['id', 'entity'],
factRows.map(it => [it.id, it.entity]),
.where({ id: factRetrieverId })
.and.whereIn('entity', db =>
db.distinct('entity').where({ id: factRetrieverId }),
)
.and.where('timestamp', '<', timestamp.toISO())
.delete();
@@ -214,46 +214,48 @@ export class TechInsightsDatabase implements TechInsightsStore {
private async deleteExpiredFactsByNumber(
tx: Transaction,
factRows: { id: string; entity: string }[],
factRetrieverId: string,
maxItems: number,
) {
const deletables = await tx<RawDbFactRow>('facts')
.whereIn(
['id', 'entity'],
factRows.map(it => [it.id, it.entity]),
)
.and.leftJoin(
joinTable =>
joinTable
.select('*')
.from(
this.db('facts')
.column(
{ fid: 'id' },
{ fentity: 'entity' },
{ ftimestamp: 'timestamp' },
)
.column(
this.db.raw(
'row_number() over (partition by id, entity order by timestamp desc) as fact_rank',
),
)
.as('ranks'),
)
.where('fact_rank', '<=', maxItems)
.as('filterjoin'),
joinClause => {
joinClause
.on('filterjoin.fid', 'facts.id')
.on('filterjoin.fentity', 'facts.entity')
.on('filterjoin.ftimestamp', 'facts.timestamp');
},
)
.whereNull('filterjoin.fid');
const deletionFilterQuery = (subTx: Knex.QueryBuilder<any, unknown[]>) =>
subTx
.select(['id', 'entity', 'timestamp'])
.from('facts')
.where({ id: factRetrieverId })
.and.whereIn('entity', db =>
db.distinct('entity').where({ id: factRetrieverId }),
)
.and.leftJoin(
joinTable =>
joinTable
.select('*')
.from(
this.db('facts')
.column(
{ fid: 'id' },
{ fentity: 'entity' },
{ ftimestamp: 'timestamp' },
)
.column(
this.db.raw(
'row_number() over (partition by id, entity order by timestamp desc) as fact_rank',
),
)
.as('ranks'),
)
.where('fact_rank', '<=', maxItems)
.as('filterjoin'),
joinClause => {
joinClause
.on('filterjoin.fid', 'facts.id')
.on('filterjoin.fentity', 'facts.entity')
.on('filterjoin.ftimestamp', 'facts.timestamp');
},
)
.whereNull('filterjoin.fid');
await tx('facts')
.whereIn(
['id', 'entity', 'timestamp'],
deletables.map(it => [it.id, it.entity, it.timestamp]),
.whereIn(['id', 'entity', 'timestamp'], database =>
deletionFilterQuery(database),
)
.delete();
}