Merge pull request #8845 from RoadieHQ/fact-garbage-truck

[TechInsights] Add expiration configurations to fact retrievers

.changeset/six-coins-admire.md

@@ -0,0 +1,40 @@
+---
+'@backstage/plugin-tech-insights-backend': minor
+'@backstage/plugin-tech-insights-node': minor
+---
+
+BREAKING CHANGES:
+
+- The helper function to create a fact retriever registration is now expecting an object of configuration items instead of individual arguments.
+  Modify your `techInsights.ts` plugin configuration in `packages/backend/src/plugins/techInsights.ts` (or equivalent) the following way:
+
+```diff
+-createFactRetrieverRegistration(
+-  '1 1 1 * *', // Example cron, At 01:01 on day-of-month 1.
+-  entityOwnershipFactRetriever,
+-),
++createFactRetrieverRegistration({
++  cadende: '1 1 1 * *', // Example cron, At 01:01 on day-of-month 1.
++  factRetriever: entityOwnershipFactRetriever,
++}),
+
+```
+
+- `TechInsightsStore` interface has changed its signature of `insertFacts` method. If you have created your own implementation of either `TechInsightsDatabase` or `FactRetrieverEngine` you need to modify the implementation/call to this method to accept/pass-in an object instead if individual arguments. The interface now accepts an additional `lifecycle` argument which is optional (defined below). An example modification to fact retriever engine:
+
+```diff
+-await this.repository.insertFacts(factRetriever.id, facts);
++await this.repository.insertFacts({
++ id: factRetriever.id,
++ facts,
++ lifecycle,
++});
+```
+
+Adds a configuration option to fact retrievers to define lifecycle for facts the retriever persists. Possible values are either 'max items' or 'time-to-live'. The former will keep only n number of items in the database for each fact per entity. The latter will remove all facts that are older than the TTL value.
+
+Possible values:
+
+- `{ maxItems: 5 }` // Deletes all facts for the retriever/entity pair, apart from the last five
+- `{ ttl: 1209600000 }` // (2 weeks) Deletes all facts older than 2 weeks for the retriever/entity pair
+- `{ ttl: { weeks: 2 } }` // Deletes all facts older than 2 weeks for the retriever/entity pair

packages/backend/src/plugins/techInsights.ts

@@ -40,12 +40,18 @@ export default async function createPlugin({
     database,
     discovery,
     factRetrievers: [
-      createFactRetrieverRegistration(
-        '1 1 1 * *', // Example cron, At 01:01 on day-of-month 1.
-        entityOwnershipFactRetriever,
-      ),
-      createFactRetrieverRegistration('1 1 1 * *', entityMetadataFactRetriever),
-      createFactRetrieverRegistration('1 1 1 * *', techdocsFactRetriever),
+      createFactRetrieverRegistration({
+        cadence: '1 1 1 * *', // Example cron, At 01:01 on day-of-month 1.
+        factRetriever: entityOwnershipFactRetriever,
+      }),
+      createFactRetrieverRegistration({
+        cadence: '1 1 1 * *',
+        factRetriever: entityMetadataFactRetriever,
+      }),
+      createFactRetrieverRegistration({
+        cadence: '1 1 1 * *',
+        factRetriever: techdocsFactRetriever,
+      }),
     ],
     factCheckerFactory: new JsonRulesEngineFactCheckerFactory({
       checks: [

plugins/tech-insights-backend/README.md

@@ -91,7 +91,15 @@ const myFactRetrieverRegistration = createFactRetrieverRegistration(
 );
 ```
 
-Then you can modify the example `techInsights.ts` file shown above like this:
+FactRetrieverRegistration also accepts an optional `lifecycle` configuration value. This can be either MaxItems or TTL (time to live). Valid options for this value are either a number for MaxItems or a Luxon duration like object for TTL. For example:
+
+```ts
+const maxItems = { maxItems: 7 }; // Deletes all but 7 latest facts for each id/entity pair
+const ttl = { ttl: 1209600000 }; // (2 weeks) Deletes items older than 2 weeks
+const ttlWithAHumanReadableValue = { ttl: { weeks: 2 } }; // Deletes items older than 2 weeks
+```
+
+To register these fact retrievers to your application you can modify the example `techInsights.ts` file shown above like this:
 
 ```diff
 const builder = new DefaultTechInsightsBuilder({

plugins/tech-insights-backend/api-report.md

@@ -8,6 +8,7 @@ import { Config } from '@backstage/config';
 import express from 'express';
 import { FactChecker } from '@backstage/plugin-tech-insights-node';
 import { FactCheckerFactory } from '@backstage/plugin-tech-insights-node';
+import { FactLifecycle } from '@backstage/plugin-tech-insights-node';
 import { FactRetriever } from '@backstage/plugin-tech-insights-node';
 import { FactRetrieverRegistration } from '@backstage/plugin-tech-insights-node';
 import { Logger as Logger_2 } from 'winston';
@@ -25,10 +26,11 @@ export const buildTechInsightsContext: <
 ) => Promise<TechInsightsContext<CheckType, CheckResultType>>;
 
 // @public
-export function createFactRetrieverRegistration(
-  cadence: string,
-  factRetriever: FactRetriever,
-): FactRetrieverRegistration;
+export function createFactRetrieverRegistration({
+  cadence,
+  factRetriever,
+  lifecycle,
+}: FactRetrieverRegistrationOptions): FactRetrieverRegistration;
 
 // @public
 export function createRouter<
@@ -42,6 +44,13 @@ export const entityMetadataFactRetriever: FactRetriever;
 // @public
 export const entityOwnershipFactRetriever: FactRetriever;
 
+// @public (undocumented)
+export type FactRetrieverRegistrationOptions = {
+  cadence: string;
+  factRetriever: FactRetriever;
+  lifecycle?: FactLifecycle;
+};
+
 // @public
 export type PersistenceContext = {
   techInsightsStore: TechInsightsStore;

plugins/tech-insights-backend/src/index.ts

@@ -25,4 +25,5 @@ export type {
 
 export type { PersistenceContext } from './service/persistence/persistenceContext';
 export { createFactRetrieverRegistration } from './service/fact/createFactRetriever';
+export type { FactRetrieverRegistrationOptions } from './service/fact/createFactRetriever';
 export * from './service/fact/factRetrievers';

plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 import {
+  FactLifecycle,
   FactRetriever,
   FactRetrieverContext,
   TechInsightFact,
@@ -76,7 +77,7 @@ export class FactRetrieverEngine {
     const registrations = this.factRetrieverRegistry.listRegistrations();
     const newRegs: string[] = [];
     registrations.forEach(registration => {
-      const { factRetriever, cadence } = registration;
+      const { factRetriever, cadence, lifecycle } = registration;
       if (!this.scheduledJobs.has(factRetriever.id)) {
         const cronExpression =
           cadence || this.defaultCadence || randomDailyCron();
@@ -88,7 +89,7 @@ export class FactRetrieverEngine {
         }
         const job = schedule(
           cronExpression,
-          this.createFactRetrieverHandler(factRetriever),
+          this.createFactRetrieverHandler(factRetriever, lifecycle),
         );
         this.scheduledJobs.set(factRetriever.id, job);
         newRegs.push(factRetriever.id);
@@ -103,7 +104,10 @@ export class FactRetrieverEngine {
     return this.scheduledJobs.get(ref);
   }
 
-  private createFactRetrieverHandler(factRetriever: FactRetriever) {
+  private createFactRetrieverHandler(
+    factRetriever: FactRetriever,
+    lifecycle?: FactLifecycle,
+  ) {
     return async () => {
       const startTimestamp = process.hrtime();
       this.logger.info(
@@ -129,7 +133,11 @@ export class FactRetrieverEngine {
       }
 
       try {
-        await this.repository.insertFacts(factRetriever.id, facts);
+        await this.repository.insertFacts({
+          id: factRetriever.id,
+          facts,
+          lifecycle,
+        });
         this.logger.info(
           `Stored ${facts.length} facts for fact retriever ${
             factRetriever.id

plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts

@@ -14,10 +14,25 @@
  * limitations under the License.
  */
 import {
+  FactLifecycle,
   FactRetriever,
   FactRetrieverRegistration,
 } from '@backstage/plugin-tech-insights-node';
 
+/**
+ * @public
+ *
+ * @param cadence - cron expression to indicate when the fact retriever should be triggered
+ * @param factRetriever - Implementation of fact retriever consisting of at least id, version, schema and handler
+ * @param lifecycle - Optional lifecycle definition indicating the cleanup logic of facts when this retriever is run
+ *
+ */
+export type FactRetrieverRegistrationOptions = {
+  cadence: string;
+  factRetriever: FactRetriever;
+  lifecycle?: FactLifecycle;
+};
+
 /**
  * @public
  *
@@ -25,6 +40,7 @@ import {
  *
  * @param cadence - cron expression to indicate when the fact retriever should be triggered
  * @param factRetriever - Implementation of fact retriever consisting of at least id, version, schema and handler
+ * @param lifecycle - Optional lifecycle definition indicating the cleanup logic of facts when this retriever is run
  *
  *
  * @remarks
@@ -40,13 +56,19 @@ import {
  # │ │ │ │ │ │
  # * * * * * *
  *
+ * Valid lifecycle values:
+ * \{ ttl: \{ weeks: 2 \} \} -- This fact retriever will remove items that are older than 2 weeks when it is run
+ * \{ maxItems: 7 \} -- This fact retriever will leave 7 newest items in the database when it is run
+ *
  */
-export function createFactRetrieverRegistration(
-  cadence: string,
-  factRetriever: FactRetriever,
-): FactRetrieverRegistration {
+export function createFactRetrieverRegistration({
+  cadence,
+  factRetriever,
+  lifecycle,
+}: FactRetrieverRegistrationOptions): FactRetrieverRegistration {
   return {
     cadence,
     factRetriever,
+    lifecycle,
   };
 }

plugins/tech-insights-backend/src/service/fact/factRetrievers/utils.ts

@@ -16,9 +16,22 @@
 import camelCase from 'lodash/camelCase';
 import { Entity } from '@backstage/catalog-model';
 import { get } from 'lodash';
+import {
+  FactLifecycle,
+  MaxItems,
+  TTL,
+} from '@backstage/plugin-tech-insights-node';
 
 export const generateAnnotationFactName = (annotation: string) =>
   camelCase(`hasAnnotation-${annotation}`);
 
 export const entityHasAnnotation = (entity: Entity, annotation: string) =>
   Boolean(get(entity, ['metadata', 'annotations', annotation]));
+
+export const isTtl = (lifecycle: FactLifecycle): lifecycle is TTL => {
+  return !!(lifecycle as TTL).timeToLive;
+};
+
+export const isMaxItems = (lifecycle: FactLifecycle): lifecycle is MaxItems => {
+  return !!(lifecycle as MaxItems).maxItems;
+};

plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts

@@ -126,10 +126,15 @@ describe('Tech Insights database', () => {
         logger: getVoidLogger(),
       })
     ).techInsightsStore;
-
+  });
+  beforeEach(async () => {
     await testDbClient.batchInsert('fact_schemas', factSchemas);
     await testDbClient.batchInsert('facts', facts);
   });
+  afterEach(async () => {
+    await testDbClient('facts').delete();
+    await testDbClient('fact_schemas').delete();
+  });
 
   const baseAssertionFact = {
     id: 'test-fact',
@@ -183,16 +188,12 @@ describe('Tech Insights database', () => {
     expect(schemas).toHaveLength(2);
     expect(schemas[0]).toMatchObject({
       id: 'test-fact',
-      version: '1.2.1-test',
+      version: '0.0.1-test',
       entityFilter: [{ kind: 'component' }],
       testNumberFact: {
         type: 'integer',
         description: 'Test fact with a number type',
       },
-      testStringFact: {
-        type: 'string',
-        description: 'Test fact with a string type',
-      },
     });
     expect(schemas[1]).toMatchObject({
       id: 'second',
@@ -263,4 +264,268 @@ describe('Tech Insights database', () => {
       facts: { testNumberFact: 3 },
     });
   });
+
+  it('should delete extraneous rows when MaxItems is defined. Should leave only n latest', async () => {
+    const deviledFact = (it: {}) => ({
+      ...it,
+      facts: JSON.stringify({
+        testNumberFact: 666,
+      }),
+    });
+    await testDbClient.batchInsert('facts', additionalFacts.map(deviledFact));
+
+    const preInsertionFacts = await testDbClient('facts').select();
+    expect(preInsertionFacts).toHaveLength(3);
+
+    const timestamp = DateTime.now().plus(Duration.fromMillis(1111));
+    const factToBeInserted = {
+      timestamp: timestamp,
+      entity: {
+        namespace: 'a',
+        kind: 'a',
+        name: 'a',
+      },
+      facts: {
+        testNumberFact: 555,
+      },
+    };
+    const maxItems = 2;
+    await store.insertFacts({
+      id: 'test-fact',
+      facts: [factToBeInserted],
+      lifecycle: { maxItems },
+    });
+
+    const afterInsertionFacts = await testDbClient('facts').select();
+    expect(afterInsertionFacts).toHaveLength(maxItems);
+    expect(afterInsertionFacts[0]).toMatchObject(
+      deviledFact(additionalFacts[0]),
+    );
+    expect(afterInsertionFacts[1]).toMatchObject({
+      id: 'test-fact',
+      version: '0.0.1-test',
+      timestamp: timestamp.toISO(),
+      entity: 'a:a/a',
+      facts: JSON.stringify({ testNumberFact: 555 }),
+    });
+  });
+
+  it('should delete extraneous rows for each entity when MaxItems is defined. Should leave only n latest', async () => {
+    const deviledFact = (it: {}) => ({
+      ...it,
+      facts: JSON.stringify({
+        testNumberFact: 666,
+      }),
+    });
+    await testDbClient.batchInsert('facts', additionalFacts.map(deviledFact));
+
+    const preInsertionFacts = await testDbClient('facts').select();
+
+    expect(preInsertionFacts).toHaveLength(3);
+
+    await testDbClient.batchInsert(
+      'facts',
+      preInsertionFacts.map(it => ({ ...it, entity: 'b:b/b' })),
+    );
+
+    const timestamp = DateTime.now().plus(Duration.fromMillis(1111));
+    const factsToBeInserted = [
+      {
+        timestamp: timestamp,
+        entity: {
+          namespace: 'a',
+          kind: 'a',
+          name: 'a',
+        },
+        facts: {
+          testNumberFact: 555,
+        },
+      },
+      {
+        timestamp: timestamp,
+        entity: {
+          namespace: 'b',
+          kind: 'b',
+          name: 'b',
+        },
+        facts: {
+          testNumberFact: 555,
+        },
+      },
+    ];
+    const maxItems = 2;
+    await store.insertFacts({
+      id: 'test-fact',
+      facts: factsToBeInserted,
+      lifecycle: { maxItems },
+    });
+
+    const afterInsertionFacts = await testDbClient('facts').select();
+
+    const inserted = {
+      id: 'test-fact',
+      version: '0.0.1-test',
+      timestamp: timestamp.toISO(),
+      entity: 'a:a/a',
+      facts: JSON.stringify({ testNumberFact: 555 }),
+    };
+    expect(afterInsertionFacts).toHaveLength(maxItems * 2);
+    expect(afterInsertionFacts[0]).toMatchObject(
+      deviledFact(additionalFacts[0]),
+    );
+
+    expect(afterInsertionFacts[1]).toMatchObject({
+      ...deviledFact(additionalFacts[0]),
+      entity: 'b:b/b',
+    });
+    expect(afterInsertionFacts[2]).toMatchObject(inserted);
+    expect(afterInsertionFacts[3]).toMatchObject({
+      ...inserted,
+      entity: 'b:b/b',
+    });
+  });
+
+  it('should delete extraneous rows when TTL is defined. Should leave only items with timestamp greater than TTL', async () => {
+    const oldStaledOutFact = (it: {}) => ({
+      ...it,
+      facts: JSON.stringify({
+        testNumberFact: 666,
+      }),
+      timestamp: DateTime.now().minus({ weeks: 3 }).toISO(),
+    });
+    await testDbClient.batchInsert(
+      'facts',
+      additionalFacts.map(oldStaledOutFact),
+    );
+
+    const preInsertionFacts = await testDbClient('facts').select();
+    expect(preInsertionFacts).toHaveLength(3);
+
+    const timestamp = DateTime.now().plus(Duration.fromMillis(1111));
+    const factToBeInserted = {
+      timestamp: timestamp,
+      entity: {
+        namespace: 'a',
+        kind: 'a',
+        name: 'a',
+      },
+      facts: {
+        testNumberFact: 555,
+      },
+    };
+    await store.insertFacts({
+      id: 'test-fact',
+      facts: [factToBeInserted],
+      lifecycle: { timeToLive: { weeks: 2 } },
+    });
+
+    const afterInsertionFacts = await testDbClient('facts')
+      .select()
+      .orderBy('timestamp', 'desc');
+    expect(afterInsertionFacts).toHaveLength(3);
+    expect(afterInsertionFacts[0]).toMatchObject({
+      id: 'test-fact',
+      version: '0.0.1-test',
+      timestamp: timestamp.toISO(),
+      entity: 'a:a/a',
+      facts: JSON.stringify({ testNumberFact: 555 }),
+    });
+    expect(afterInsertionFacts[1]).toMatchObject(facts[1]);
+    expect(afterInsertionFacts[2]).toMatchObject(facts[0]);
+
+    expect(afterInsertionFacts).not.toContainEqual(
+      oldStaledOutFact(additionalFacts[0]),
+    );
+  });
+
+  it('should delete extraneous rows for each entity when TTL expired', async () => {
+    const oldStaledOutFact = (it: {}) => ({
+      ...it,
+      facts: JSON.stringify({
+        testNumberFact: 666,
+      }),
+      timestamp: DateTime.now().minus({ weeks: 3 }).toISO(),
+    });
+    await testDbClient.batchInsert(
+      'facts',
+      additionalFacts.map(oldStaledOutFact),
+    );
+
+    const preInsertionFacts = await testDbClient('facts').select();
+    expect(preInsertionFacts).toHaveLength(3);
+
+    await testDbClient.batchInsert(
+      'facts',
+      preInsertionFacts.map(it => ({ ...it, entity: 'b:b/b' })),
+    );
+
+    const timestamp = DateTime.now().plus(Duration.fromMillis(1111));
+    const factsToBeInserted = [
+      {
+        timestamp: timestamp,
+        entity: {
+          namespace: 'a',
+          kind: 'a',
+          name: 'a',
+        },
+        facts: {
+          testNumberFact: 555,
+        },
+      },
+      {
+        timestamp: timestamp,
+        entity: {
+          namespace: 'b',
+          kind: 'b',
+          name: 'b',
+        },
+        facts: {
+          testNumberFact: 555,
+        },
+      },
+    ];
+    await store.insertFacts({
+      id: 'test-fact',
+      facts: factsToBeInserted,
+      lifecycle: { timeToLive: { weeks: 2 } },
+    });
+
+    const afterInsertionFacts = await testDbClient('facts')
+      .select()
+      .orderBy('timestamp', 'desc');
+
+    expect(afterInsertionFacts).toHaveLength(6);
+    expect(afterInsertionFacts[0]).toMatchObject({
+      id: 'test-fact',
+      version: '0.0.1-test',
+      timestamp: timestamp.toISO(),
+      entity: 'a:a/a',
+      facts: JSON.stringify({ testNumberFact: 555 }),
+    });
+    expect(afterInsertionFacts[1]).toMatchObject({
+      id: 'test-fact',
+      version: '0.0.1-test',
+      timestamp: timestamp.toISO(),
+      entity: 'b:b/b',
+      facts: JSON.stringify({ testNumberFact: 555 }),
+    });
+    expect(afterInsertionFacts[2]).toMatchObject(facts[1]);
+    expect(afterInsertionFacts[3]).toMatchObject({
+      ...facts[1],
+      entity: 'b:b/b',
+    });
+    expect(afterInsertionFacts[4]).toMatchObject(facts[0]);
+    expect(afterInsertionFacts[5]).toMatchObject({
+      ...facts[0],
+      entity: 'b:b/b',
+    });
+
+    expect(afterInsertionFacts).not.toContainEqual(
+      oldStaledOutFact(additionalFacts[0]),
+    );
+    expect(afterInsertionFacts).not.toContainEqual({
+      ...oldStaledOutFact(additionalFacts[0]),
+      entity: 'b:b/b',
+    });
+  });
 });

plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts

@@ -15,17 +15,20 @@
  */
 import { Knex } from 'knex';
 import {
+  FactLifecycle,
   FactSchema,
-  TechInsightFact,
-  FlatTechInsightFact,
-  TechInsightsStore,
   FactSchemaDefinition,
+  FlatTechInsightFact,
+  TechInsightFact,
+  TechInsightsStore,
 } from '@backstage/plugin-tech-insights-node';
 import { rsort } from 'semver';
 import { groupBy, omit } from 'lodash';
 import { DateTime } from 'luxon';
 import { Logger } from 'winston';
 import { parseEntityName, stringifyEntityRef } from '@backstage/catalog-model';
+import { isMaxItems, isTtl } from '../fact/factRetrievers/utils';
+import Transaction = Knex.Transaction;
 
 export type RawDbFactRow = {
   id: string;
@@ -84,7 +87,15 @@ export class TechInsightsDatabase implements TechInsightsStore {
     }
   }
 
-  async insertFacts(id: string, facts: TechInsightFact[]): Promise<void> {
+  async insertFacts({
+    id,
+    facts,
+    lifecycle,
+  }: {
+    id: string;
+    facts: TechInsightFact[];
+    lifecycle?: FactLifecycle;
+  }): Promise<void> {
     if (facts.length === 0) return;
     const currentSchema = await this.getLatestSchema(id);
     const factRows = facts.map(it => {
@@ -93,11 +104,20 @@ export class TechInsightsDatabase implements TechInsightsStore {
         version: currentSchema.version,
         entity: stringifyEntityRef(it.entity),
         facts: JSON.stringify(it.facts),
-        ...(it.timestamp && { timestamp: it.timestamp.toJSDate() }),
+        ...(it.timestamp && { timestamp: it.timestamp.toISO() }),
       };
     });
+
     await this.db.transaction(async tx => {
       await tx.batchInsert<RawDbFactRow>('facts', factRows, this.CHUNK_SIZE);
+
+      if (lifecycle && isTtl(lifecycle)) {
+        const expiration = DateTime.now().minus(lifecycle.timeToLive);
+        await this.deleteExpiredFactsByDate(tx, factRows, expiration);
+      }
+      if (lifecycle && isMaxItems(lifecycle)) {
+        await this.deleteExpiredFactsByNumber(tx, factRows, lifecycle.maxItems);
+      }
     });
   }
 
@@ -170,6 +190,66 @@ export class TechInsightsDatabase implements TechInsightsStore {
     return existingSchemas.find(it => it.version === sorted[0])!!;
   }
 
+  private async deleteExpiredFactsByDate(
+    tx: Transaction,
+    factRows: { id: string; entity: string }[],
+    timestamp: DateTime,
+  ) {
+    await tx<RawDbFactRow>('facts')
+      .whereIn(
+        ['id', 'entity'],
+        factRows.map(it => [it.id, it.entity]),
+      )
+      .and.where('timestamp', '<', timestamp.toISO())
+      .delete();
+  }
+
+  private async deleteExpiredFactsByNumber(
+    tx: Transaction,
+    factRows: { id: string; entity: string }[],
+    maxItems: number,
+  ) {
+    const deletables = await tx<RawDbFactRow>('facts')
+      .whereIn(
+        ['id', 'entity'],
+        factRows.map(it => [it.id, it.entity]),
+      )
+      .and.leftJoin(
+        joinTable =>
+          joinTable
+            .select('*')
+            .from(
+              this.db('facts')
+                .column(
+                  { fid: 'id' },
+                  { fentity: 'entity' },
+                  { ftimestamp: 'timestamp' },
+                )
+                .column(
+                  this.db.raw(
+                    'row_number() over (partition by id, entity order by timestamp desc) as fact_rank',
+                  ),
+                )
+                .as('ranks'),
+            )
+            .where('fact_rank', '<=', maxItems)
+            .as('filterjoin'),
+        joinClause => {
+          joinClause
+            .on('filterjoin.fid', 'facts.id')
+            .on('filterjoin.fentity', 'facts.entity')
+            .on('filterjoin.ftimestamp', 'facts.timestamp');
+        },
+      )
+      .whereNull('filterjoin.fid');
+    await tx('facts')
+      .whereIn(
+        ['id', 'entity', 'timestamp'],
+        deletables.map(it => [it.id, it.entity, it.timestamp]),
+      )
+      .delete();
+  }
+
   private dbFactRowsToTechInsightFacts(rows: RawDbFactRow[]) {
     return rows.reduce((acc, it) => {
       const { namespace, kind, name } = parseEntityName(it.entity);

plugins/tech-insights-node/api-report.md

@@ -6,6 +6,7 @@
 import { CheckResult } from '@backstage/plugin-tech-insights-common';
 import { Config } from '@backstage/config';
 import { DateTime } from 'luxon';
+import { DurationLike } from 'luxon';
 import { Logger as Logger_2 } from 'winston';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
 
@@ -37,6 +38,9 @@ export interface FactCheckerFactory<
   ): FactChecker<CheckType, CheckResultType>;
 }
 
+// @public
+export type FactLifecycle = TTL | MaxItems;
+
 // @public
 export interface FactRetriever {
   entityFilter?:
@@ -62,6 +66,7 @@ export type FactRetrieverContext = {
 export type FactRetrieverRegistration = {
   factRetriever: FactRetriever;
   cadence?: string;
+  lifecycle?: FactLifecycle;
 };
 
 // @public
@@ -82,6 +87,11 @@ export type FlatTechInsightFact = TechInsightFact & {
   id: string;
 };
 
+// @public
+export type MaxItems = {
+  maxItems: number;
+};
+
 // @public
 export interface TechInsightCheck {
   description: string;
@@ -144,9 +154,22 @@ export interface TechInsightsStore {
     [factRef: string]: FlatTechInsightFact;
   }>;
   getLatestSchemas(ids?: string[]): Promise<FactSchema[]>;
-  insertFacts(id: string, facts: TechInsightFact[]): Promise<void>;
+  insertFacts({
+    id,
+    facts,
+    lifecycle,
+  }: {
+    id: string;
+    facts: TechInsightFact[];
+    lifecycle?: FactLifecycle;
+  }): Promise<void>;
   insertFactSchema(schemaDefinition: FactSchemaDefinition): Promise<void>;
 }
 
+// @public
+export type TTL = {
+  timeToLive: DurationLike;
+};
+
 // (No @packageDocumentation comment for this package)
 ```

plugins/tech-insights-node/src/facts.ts

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { DateTime } from 'luxon';
+import { DateTime, DurationLike } from 'luxon';
 import { Config } from '@backstage/config';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
 import { Logger } from 'winston';
@@ -190,6 +190,36 @@ export interface FactRetriever {
     | Record<string, string | symbol | (string | symbol)[]>;
 }
 
+/**
+ * @public
+ *
+ * A Luxon duration like object for time to live value
+ *
+ * @example
+ * \{ timeToLive: 1209600000 \}
+ * \{ timeToLive: \{ weeks: 4 \} \}
+ *
+ **/
+export type TTL = { timeToLive: DurationLike };
+
+/**
+ * @public
+ *
+ * A maximum number for items to be kept in the database for each fact retriever/entity pair
+ *
+ * @example
+ * \{ maxItems: 10 \}
+ *
+ **/
+export type MaxItems = { maxItems: number };
+
+/**
+ * @public
+ *
+ * A fact lifecycle definition. Determines which strategy to use to purge expired facts from the database.
+ */
+export type FactLifecycle = TTL | MaxItems;
+
 /**
  * @public
  *
@@ -216,4 +246,11 @@ export type FactRetrieverRegistration = {
    *
    */
   cadence?: string;
+
+  /**
+   * Fact lifecycle definition
+   *
+   * If defined this value will be used to determine expired items which will deleted when this fact retriever is run
+   */
+  lifecycle?: FactLifecycle;
 };

plugins/tech-insights-node/src/persistence.ts

@@ -18,6 +18,7 @@ import {
   TechInsightFact,
   FlatTechInsightFact,
   FactSchemaDefinition,
+  FactLifecycle,
 } from './facts';
 import { DateTime } from 'luxon';
 
@@ -35,8 +36,17 @@ export interface TechInsightsStore {
    *
    * @param id - Unique identifier of the fact retriever these facts relate to
    * @param facts - A collection of TechInsightFacts
+   * @param lifecycle - (Optional) Fact lifecycle object indicating the expiration logic for these items
    */
-  insertFacts(id: string, facts: TechInsightFact[]): Promise<void>;
+  insertFacts({
+    id,
+    facts,
+    lifecycle,
+  }: {
+    id: string;
+    facts: TechInsightFact[];
+    lifecycle?: FactLifecycle;
+  }): Promise<void>;
 
   /**
    * @param ids - A collection of fact row identifiers