Modify based on review comments.
Make it breaking change. Signed-off-by: Jussi Hallila <jussi@hallila.com>
This commit is contained in:
@@ -1,12 +1,29 @@
|
||||
---
|
||||
'@backstage/plugin-tech-insights-backend': patch
|
||||
'@backstage/plugin-tech-insights-node': patch
|
||||
'@backstage/plugin-tech-insights-backend': minor
|
||||
'@backstage/plugin-tech-insights-node': minor
|
||||
---
|
||||
|
||||
Adds a configuration option to fact retrievers to define lifecycle for facts the retriever persists. Possible values are either 'items-to-live' or 'time-to-live'. The former will keep only n number of items in to the database for each fact per entity. The latter will remove all facts that are older than the TTL value.
|
||||
BREAKING CHANGES:
|
||||
|
||||
- The helper function to create a fact retriever registration is now expecting an object of configuration items instead of individual arguments.
|
||||
Modify your techInsights.ts plugin configuration in `packages/backend/src/plugins/techInsights.ts` (or equivalent) the following way:
|
||||
|
||||
```diff
|
||||
-createFactRetrieverRegistration(
|
||||
- '1 1 1 * *', // Example cron, At 01:01 on day-of-month 1.
|
||||
- entityOwnershipFactRetriever,
|
||||
-),
|
||||
+createFactRetrieverRegistration({
|
||||
+ cadende: '1 1 1 * *', // Example cron, At 01:01 on day-of-month 1.
|
||||
+ factRetriever: entityOwnershipFactRetriever,
|
||||
+}),
|
||||
|
||||
```
|
||||
|
||||
Adds a configuration option to fact retrievers to define lifecycle for facts the retriever persists. Possible values are either 'max items' or 'time-to-live'. The former will keep only n number of items in the database for each fact per entity. The latter will remove all facts that are older than the TTL value.
|
||||
|
||||
Possible values:
|
||||
|
||||
- `{ itl: 5 }` // Deletes all facts for the retriever/entity pair, apart from the last five
|
||||
- `{ maxItems: 5 }` // Deletes all facts for the retriever/entity pair, apart from the last five
|
||||
- `{ ttl: 1209600000 }` // (2 weeks) Deletes all facts older than 2 weeks for the retriever/entity pair
|
||||
- `{ ttl: { weeks: 2 } }` // Deletes all facts older than 2 weeks for the retriever/entity pair
|
||||
|
||||
@@ -40,12 +40,18 @@ export default async function createPlugin({
|
||||
database,
|
||||
discovery,
|
||||
factRetrievers: [
|
||||
createFactRetrieverRegistration(
|
||||
'1 1 1 * *', // Example cron, At 01:01 on day-of-month 1.
|
||||
entityOwnershipFactRetriever,
|
||||
),
|
||||
createFactRetrieverRegistration('1 1 1 * *', entityMetadataFactRetriever),
|
||||
createFactRetrieverRegistration('1 1 1 * *', techdocsFactRetriever),
|
||||
createFactRetrieverRegistration({
|
||||
cadence: '1 1 1 * *', // Example cron, At 01:01 on day-of-month 1.
|
||||
factRetriever: entityOwnershipFactRetriever,
|
||||
}),
|
||||
createFactRetrieverRegistration({
|
||||
cadence: '1 1 1 * *',
|
||||
factRetriever: entityMetadataFactRetriever,
|
||||
}),
|
||||
createFactRetrieverRegistration({
|
||||
cadence: '1 1 1 * *',
|
||||
factRetriever: techdocsFactRetriever,
|
||||
}),
|
||||
],
|
||||
factCheckerFactory: new JsonRulesEngineFactCheckerFactory({
|
||||
checks: [
|
||||
|
||||
@@ -91,10 +91,10 @@ const myFactRetrieverRegistration = createFactRetrieverRegistration(
|
||||
);
|
||||
```
|
||||
|
||||
FactRetrieverRegistration also accepts an optional `lifecycle` configuration value. This can be either ITL (items to live) or TTL (time to live). Valid options for this value are either a number for ITL or a Luxon duration like object for TTL. For example:
|
||||
FactRetrieverRegistration also accepts an optional `lifecycle` configuration value. This can be either MaxItems or TTL (time to live). Valid options for this value are either a number for MaxItems or a Luxon duration like object for TTL. For example:
|
||||
|
||||
```ts
|
||||
const itl = { itl: 7 }; // Deletes all but 7 latest facts for each id/entity pair
|
||||
const maxItems = { maxItems: 7 }; // Deletes all but 7 latest facts for each id/entity pair
|
||||
const ttl = { ttl: 1209600000 }; // (2 weeks) Deletes items older than 2 weeks
|
||||
const ttlWithAHumanReadableValue = { ttl: { weeks: 2 } }; // Deletes items older than 2 weeks
|
||||
```
|
||||
|
||||
@@ -19,6 +19,12 @@ import {
|
||||
FactRetrieverRegistration,
|
||||
} from '@backstage/plugin-tech-insights-node';
|
||||
|
||||
export type FactRetrieverRegistrationOptions = {
|
||||
cadence: string;
|
||||
factRetriever: FactRetriever;
|
||||
lifecycle?: FactLifecycle;
|
||||
};
|
||||
|
||||
/**
|
||||
* @public
|
||||
*
|
||||
@@ -47,11 +53,11 @@ import {
|
||||
* \{ itl: 7 \} -- This fact retriever will leave 7 newest items in the database when it is run
|
||||
*
|
||||
*/
|
||||
export function createFactRetrieverRegistration(
|
||||
cadence: string,
|
||||
factRetriever: FactRetriever,
|
||||
lifecycle?: FactLifecycle,
|
||||
): FactRetrieverRegistration {
|
||||
export function createFactRetrieverRegistration({
|
||||
cadence,
|
||||
factRetriever,
|
||||
lifecycle,
|
||||
}: FactRetrieverRegistrationOptions): FactRetrieverRegistration {
|
||||
return {
|
||||
cadence,
|
||||
factRetriever,
|
||||
|
||||
@@ -16,7 +16,11 @@
|
||||
import camelCase from 'lodash/camelCase';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { get } from 'lodash';
|
||||
import { FactLifecycle, ITL, TTL } from '@backstage/plugin-tech-insights-node';
|
||||
import {
|
||||
FactLifecycle,
|
||||
MaxItems,
|
||||
TTL,
|
||||
} from '@backstage/plugin-tech-insights-node';
|
||||
|
||||
export const generateAnnotationFactName = (annotation: string) =>
|
||||
camelCase(`hasAnnotation-${annotation}`);
|
||||
@@ -25,9 +29,9 @@ export const entityHasAnnotation = (entity: Entity, annotation: string) =>
|
||||
Boolean(get(entity, ['metadata', 'annotations', annotation]));
|
||||
|
||||
export const isTtl = (lifecycle: FactLifecycle): lifecycle is TTL => {
|
||||
return !!(lifecycle as TTL).ttl;
|
||||
return !!(lifecycle as TTL).timeToLive;
|
||||
};
|
||||
|
||||
export const isItl = (lifecycle: FactLifecycle): lifecycle is ITL => {
|
||||
return !!(lifecycle as ITL).itl;
|
||||
export const isMaxItems = (lifecycle: FactLifecycle): lifecycle is MaxItems => {
|
||||
return !!(lifecycle as MaxItems).maxItems;
|
||||
};
|
||||
|
||||
@@ -265,7 +265,7 @@ describe('Tech Insights database', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('should delete extraneous rows when ITL is defined. Should leave only n latest', async () => {
|
||||
it('should delete extraneous rows when MaxItems is defined. Should leave only n latest', async () => {
|
||||
const deviledFact = (it: {}) => ({
|
||||
...it,
|
||||
facts: JSON.stringify({
|
||||
@@ -289,11 +289,11 @@ describe('Tech Insights database', () => {
|
||||
testNumberFact: 555,
|
||||
},
|
||||
};
|
||||
const itl = 2;
|
||||
await store.insertFacts('test-fact', [factToBeInserted], { itl });
|
||||
const maxItems = 2;
|
||||
await store.insertFacts('test-fact', [factToBeInserted], { maxItems });
|
||||
|
||||
const afterInsertionFacts = await testDbClient('facts').select();
|
||||
expect(afterInsertionFacts).toHaveLength(itl);
|
||||
expect(afterInsertionFacts).toHaveLength(maxItems);
|
||||
expect(afterInsertionFacts[0]).toMatchObject(
|
||||
deviledFact(additionalFacts[0]),
|
||||
);
|
||||
@@ -335,7 +335,7 @@ describe('Tech Insights database', () => {
|
||||
},
|
||||
};
|
||||
await store.insertFacts('test-fact', [factToBeInserted], {
|
||||
ttl: { weeks: 2 },
|
||||
timeToLive: { weeks: 2 },
|
||||
});
|
||||
|
||||
const afterInsertionFacts = await testDbClient('facts')
|
||||
|
||||
@@ -27,7 +27,7 @@ import { groupBy, omit } from 'lodash';
|
||||
import { DateTime } from 'luxon';
|
||||
import { Logger } from 'winston';
|
||||
import { parseEntityName, stringifyEntityRef } from '@backstage/catalog-model';
|
||||
import { isItl, isTtl } from '../fact/factRetrievers/utils';
|
||||
import { isMaxItems, isTtl } from '../fact/factRetrievers/utils';
|
||||
import Transaction = Knex.Transaction;
|
||||
|
||||
export type RawDbFactRow = {
|
||||
@@ -108,12 +108,11 @@ export class TechInsightsDatabase implements TechInsightsStore {
|
||||
await tx.batchInsert<RawDbFactRow>('facts', factRows, this.CHUNK_SIZE);
|
||||
|
||||
if (lifecycle && isTtl(lifecycle)) {
|
||||
const expiration = DateTime.now().minus(lifecycle.ttl);
|
||||
const expiration = DateTime.now().minus(lifecycle.timeToLive);
|
||||
await this.deleteExpiredFactsByDate(tx, factRows, expiration);
|
||||
}
|
||||
if (lifecycle && isItl(lifecycle)) {
|
||||
const items = lifecycle.itl;
|
||||
await this.deleteExpiredFactsByNumber(tx, factRows, items);
|
||||
if (lifecycle && isMaxItems(lifecycle)) {
|
||||
await this.deleteExpiredFactsByNumber(tx, factRows, lifecycle.maxItems);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -204,7 +203,7 @@ export class TechInsightsDatabase implements TechInsightsStore {
|
||||
private async deleteExpiredFactsByNumber(
|
||||
tx: Transaction,
|
||||
factRows: { id: string; entity: string }[],
|
||||
items: number,
|
||||
maxItems: number,
|
||||
) {
|
||||
const deletables = await tx<RawDbFactRow>('facts')
|
||||
.whereIn(
|
||||
@@ -220,7 +219,7 @@ export class TechInsightsDatabase implements TechInsightsStore {
|
||||
row_number() over (partition by id, entity order by timestamp desc) as fact_rank
|
||||
from facts) ranks
|
||||
where fact_rank <= ?? ) as filterjoin`,
|
||||
items,
|
||||
maxItems,
|
||||
),
|
||||
joinClause => {
|
||||
joinClause
|
||||
|
||||
@@ -196,29 +196,29 @@ export interface FactRetriever {
|
||||
* A Luxon duration like object for time to live value
|
||||
*
|
||||
* @example
|
||||
* \{ ttl: 1209600000 \}
|
||||
* \{ ttl: \{ weeks: 4 \} \}
|
||||
* \{ timeToLive: 1209600000 \}
|
||||
* \{ timeToLive: \{ weeks: 4 \} \}
|
||||
*
|
||||
**/
|
||||
export type TTL = { ttl: DurationLike };
|
||||
export type TTL = { timeToLive: DurationLike };
|
||||
|
||||
/**
|
||||
* @public
|
||||
*
|
||||
* A number for items to live value
|
||||
* A maximum number for items to be kept in the database for each fact retriever/entity pair
|
||||
*
|
||||
* @example
|
||||
* \{ itl: 10 \}
|
||||
* \{ maxItems: 10 \}
|
||||
*
|
||||
**/
|
||||
export type ITL = { itl: number };
|
||||
export type MaxItems = { maxItems: number };
|
||||
|
||||
/**
|
||||
* @public
|
||||
*
|
||||
* A fact lifecycle definition. Determines which strategy to use to purge expired facts from the database.
|
||||
*/
|
||||
export type FactLifecycle = TTL | ITL;
|
||||
export type FactLifecycle = TTL | MaxItems;
|
||||
|
||||
/**
|
||||
* @public
|
||||
|
||||
Reference in New Issue
Block a user