diff --git a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts index bd330bd378..0b81ad490f 100644 --- a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts +++ b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts @@ -310,6 +310,81 @@ describe('Tech Insights database', () => { }); }); + it('should delete extraneous rows for each entity when MaxItems is defined. Should leave only n latest', async () => { + const deviledFact = (it: {}) => ({ + ...it, + facts: JSON.stringify({ + testNumberFact: 666, + }), + }); + await testDbClient.batchInsert('facts', additionalFacts.map(deviledFact)); + + const preInsertionFacts = await testDbClient('facts').select(); + + expect(preInsertionFacts).toHaveLength(3); + + await testDbClient.batchInsert( + 'facts', + preInsertionFacts.map(it => ({ ...it, entity: 'b:b/b' })), + ); + + const timestamp = DateTime.now().plus(Duration.fromMillis(1111)); + const factsToBeInserted = [ + { + timestamp: timestamp, + entity: { + namespace: 'a', + kind: 'a', + name: 'a', + }, + facts: { + testNumberFact: 555, + }, + }, + { + timestamp: timestamp, + entity: { + namespace: 'b', + kind: 'b', + name: 'b', + }, + facts: { + testNumberFact: 555, + }, + }, + ]; + const maxItems = 2; + await store.insertFacts({ + id: 'test-fact', + facts: factsToBeInserted, + lifecycle: { maxItems }, + }); + + const afterInsertionFacts = await testDbClient('facts').select(); + + const inserted = { + id: 'test-fact', + version: '0.0.1-test', + timestamp: timestamp.toISO(), + entity: 'a:a/a', + facts: JSON.stringify({ testNumberFact: 555 }), + }; + expect(afterInsertionFacts).toHaveLength(maxItems * 2); + expect(afterInsertionFacts[0]).toMatchObject( + deviledFact(additionalFacts[0]), + ); + + expect(afterInsertionFacts[1]).toMatchObject({ + ...deviledFact(additionalFacts[0]), + entity: 'b:b/b', + }); + expect(afterInsertionFacts[2]).toMatchObject(inserted); + expect(afterInsertionFacts[3]).toMatchObject({ + ...inserted, + entity: 'b:b/b', + }); + }); + it('should delete extraneous rows when TTL is defined. Should leave only items with timestamp greater than TTL', async () => { const oldStaledOutFact = (it: {}) => ({ ...it, @@ -362,4 +437,95 @@ describe('Tech Insights database', () => { oldStaledOutFact(additionalFacts[0]), ); }); + + it('should delete extraneous rows for each entity when TTL expired', async () => { + const oldStaledOutFact = (it: {}) => ({ + ...it, + facts: JSON.stringify({ + testNumberFact: 666, + }), + timestamp: DateTime.now().minus({ weeks: 3 }).toISO(), + }); + await testDbClient.batchInsert( + 'facts', + additionalFacts.map(oldStaledOutFact), + ); + + const preInsertionFacts = await testDbClient('facts').select(); + expect(preInsertionFacts).toHaveLength(3); + + await testDbClient.batchInsert( + 'facts', + preInsertionFacts.map(it => ({ ...it, entity: 'b:b/b' })), + ); + + const timestamp = DateTime.now().plus(Duration.fromMillis(1111)); + const factsToBeInserted = [ + { + timestamp: timestamp, + entity: { + namespace: 'a', + kind: 'a', + name: 'a', + }, + facts: { + testNumberFact: 555, + }, + }, + { + timestamp: timestamp, + entity: { + namespace: 'b', + kind: 'b', + name: 'b', + }, + facts: { + testNumberFact: 555, + }, + }, + ]; + await store.insertFacts({ + id: 'test-fact', + facts: factsToBeInserted, + lifecycle: { timeToLive: { weeks: 2 } }, + }); + + const afterInsertionFacts = await testDbClient('facts') + .select() + .orderBy('timestamp', 'desc'); + + expect(afterInsertionFacts).toHaveLength(6); + expect(afterInsertionFacts[0]).toMatchObject({ + id: 'test-fact', + version: '0.0.1-test', + timestamp: timestamp.toISO(), + entity: 'a:a/a', + facts: JSON.stringify({ testNumberFact: 555 }), + }); + expect(afterInsertionFacts[1]).toMatchObject({ + id: 'test-fact', + version: '0.0.1-test', + timestamp: timestamp.toISO(), + entity: 'b:b/b', + facts: JSON.stringify({ testNumberFact: 555 }), + }); + expect(afterInsertionFacts[2]).toMatchObject(facts[1]); + expect(afterInsertionFacts[3]).toMatchObject({ + ...facts[1], + entity: 'b:b/b', + }); + expect(afterInsertionFacts[4]).toMatchObject(facts[0]); + expect(afterInsertionFacts[5]).toMatchObject({ + ...facts[0], + entity: 'b:b/b', + }); + + expect(afterInsertionFacts).not.toContainEqual( + oldStaledOutFact(additionalFacts[0]), + ); + expect(afterInsertionFacts).not.toContainEqual({ + ...oldStaledOutFact(additionalFacts[0]), + entity: 'b:b/b', + }); + }); }); diff --git a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts index be761ff555..c6dda440fb 100644 --- a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts +++ b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts @@ -215,16 +215,25 @@ export class TechInsightsDatabase implements TechInsightsStore { factRows.map(it => [it.id, it.entity]), ) .and.leftJoin( - this.db.raw( - `(select * - from (select id fid, - entity fentity, - timestamp ftimestamp, - row_number() over (partition by id, entity order by timestamp desc) as fact_rank - from facts) ranks - where fact_rank <= ?? ) as filterjoin`, - maxItems, - ), + joinTable => + joinTable + .select('*') + .from( + this.db('facts') + .column( + { fid: 'id' }, + { fentity: 'entity' }, + { ftimestamp: 'timestamp' }, + ) + .column( + this.db.raw( + 'row_number() over (partition by id, entity order by timestamp desc) as fact_rank', + ), + ) + .as('ranks'), + ) + .where('fact_rank', '<=', maxItems) + .as('filterjoin'), joinClause => { joinClause .on('filterjoin.fid', 'facts.id') @@ -233,7 +242,6 @@ export class TechInsightsDatabase implements TechInsightsStore { }, ) .whereNull('filterjoin.fid'); - await tx('facts') .whereIn( ['id', 'entity', 'timestamp'],