paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 33 Packages Projects Releases Wiki Activity

Address various code review comments.

Browse Source 
Signed-off-by: Jussi Hallila <jussi@hallila.com>
This commit is contained in:
Jussi Hallila
2021-10-25 17:04:33 +02:00
parent a1afbe0498
commit df000b9596
31 changed files with 529 additions and 473 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/tech-insights-backend/CHANGELOG.md
-7
View File
@@ -1,7 +0,0 @@
# @backstage/plugin-tech-insights-backend
## 0.0.1
### Patch Changes
- Initial implementation
plugins/tech-insights-backend/README.md
+1 -1
View File
@@ -124,7 +124,7 @@ const myFactRetriever: FactRetriever = {
examplenumberfact: {
type: 'integer', // Type of the fact
description: 'A fact of a number', // Description of the fact
entityKinds: ['component'], // An array of entity kinds that this fact is applicable to
entityTypes: ['component'], // An array of entity kinds that this fact is applicable to
},
},
},
plugins/tech-insights-backend/api-report.md
+10 -9
View File
@@ -15,21 +15,22 @@ import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { TechInsightCheck } from '@backstage/plugin-tech-insights-common';
import { TechInsightsStore } from '@backstage/plugin-tech-insights-common';
// Warning: (ae-missing-release-tag) "buildTechInsightsContext" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
export const buildTechInsightsContext: <
CheckType extends TechInsightCheck,
CheckResultType extends CheckResult,
>(
options: TechInsightsOptions<CheckType, CheckResultType>,
) => Promise<TechInsightsContext<CheckType, CheckResultType>>;
// @public
export function createRouter<
CheckType extends TechInsightCheck,
CheckResultType extends CheckResult,
>(options: RouterOptions<CheckType, CheckResultType>): Promise<express.Router>;
// @public (undocumented)
export class DefaultTechInsightsBuilder<
CheckType extends TechInsightCheck,
CheckResultType extends CheckResult,
> {
constructor(options: TechInsightsOptions<CheckType, CheckResultType>);
build(): Promise<TechInsightsContext<CheckType, CheckResultType>>;
}
// @public
export type PersistenceContext = {
techInsightsStore: TechInsightsStore;
plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js
+10 -7
View File
@@ -24,24 +24,28 @@ exports.up = async function up(knex) {
table.comment(
'The table for tech insight fact schemas. Containing a versioned data model definition for a collection of facts.',
);
table.increments('id').primary();
table
.text('ref')
.text('id')
.notNullable()
.comment('Identifier of the fact retriever plugin/package');
table
.string('version')
.notNullable()
.comment('SemVer string defining the version of schema.');
table
.string('entityTypes')
.nullable()
.comment(
'A comma separated collection of entity kinds the fact retriever providing this schema affects. Defaults to null, which means all entity kinds.',
);
table
.text('schema')
.notNullable()
.comment(
'Fact schema defining the values/types what this version of the fact would contain.',
);
table.index('ref', 'fact_schema_ref_idx');
table.index(['ref', 'version'], 'fact_schema_ref_version_idx');
table.primary(['id', 'version']);
table.index('id', 'fact_schema_id_idx');
});
};
@@ -50,8 +54,7 @@ exports.up = async function up(knex) {
*/
exports.down = async function down(knex) {
await knex.schema.alterTable('fact_schemas', table => {
table.dropIndex([], 'fact_schema_ref_idx');
table.dropIndex([], 'fact_schema_ref_version_idx');
table.dropIndex([], 'fact_schema_id_idx');
});
await knex.schema.dropTable('fact_schemas');
};
plugins/tech-insights-backend/migrations/202109061212_facts.js
+10 -11
View File
@@ -25,11 +25,7 @@ exports.up = async function up(knex) {
'The table for tech insight fact collections. Contains facts for individual fact retriever namespace/ref.',
);
table
.bigIncrements('index')
.notNullable()
.comment('An insert counter to ensure ordering');
table
.text('ref')
.text('id')
.notNullable()
.comment('Unique identifier of the fact retriever plugin/package');
table
@@ -54,9 +50,13 @@ exports.up = async function up(knex) {
'Values of the fact collection stored as key-value pairs in JSON format.',
);
table.index('index', 'fact_index_idx');
table.index('ref', 'fact_ref_idx');
table.index(['ref', 'entity'], 'fact_ref_entity_idx');
table
.foreign(['id', 'version'])
.references(['id', 'version'])
.inTable('fact_schemas');
table.index(['id', 'entity'], 'fact_id_entity_idx');
table.index('id', 'fact_id_idx');
});
};
@@ -65,9 +65,8 @@ exports.up = async function up(knex) {
*/
exports.down = async function down(knex) {
await knex.schema.alterTable('facts', table => {
table.dropIndex([], 'facts_index_idx');
table.dropIndex([], 'fact_ref_idx');
table.dropIndex([], 'fact_ref_entity_idx');
table.dropIndex([], 'fact_id_idx');
table.dropIndex([], 'fact_id_entity_idx');
});
await knex.schema.dropTable('facts');
};
plugins/tech-insights-backend/src/index.ts
+2 -2
View File
@@ -17,10 +17,10 @@
export * from './service/router';
export type { RouterOptions } from './service/router';
export { DefaultTechInsightsBuilder } from './service/DefaultTechInsightsBuilder';
export { buildTechInsightsContext } from './service/techInsightsContextBuilder';
export type {
TechInsightsOptions,
TechInsightsContext,
} from './service/DefaultTechInsightsBuilder';
} from './service/techInsightsContextBuilder';
export type { PersistenceContext } from './service/persistence/DatabaseManager';
plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts
+21 -24
View File
@@ -16,7 +16,7 @@
import {
FactRetriever,
FactRetrieverRegistration,
FactSchema,
FactSchemaDefinition,
TechInsightFact,
TechInsightsStore,
} from '@backstage/plugin-tech-insights-common';
@@ -35,21 +35,18 @@ jest.mock('node-cron', () => {
});
const testFactRetriever: FactRetriever = {
ref: 'test-factretriever',
id: 'test-factretriever',
version: '0.0.1',
entityTypes: ['component'],
schema: {
version: '0.0.1',
schema: {
testnumberfact: {
type: 'integer',
description: '',
entityKinds: ['component'],
},
testnumberfact: {
type: 'integer',
description: '',
},
},
handler: async () => {
return [
{
ref: 'test-factretriever',
entity: {
namespace: 'a',
kind: 'a',
@@ -65,7 +62,9 @@ const testFactRetriever: FactRetriever = {
const cadence = '1 * * * *';
describe('FactRetrieverEngine', () => {
let engine: FactRetrieverEngine;
let factSchemaAssertionCallback: (ref: string, schema: FactSchema) => void;
let factSchemaAssertionCallback: (
factSchemaDefinition: FactSchemaDefinition,
) => void;
let factInsertionAssertionCallback: (facts: TechInsightFact[]) => void;
const mockRepository: TechInsightsStore = {
@@ -73,8 +72,8 @@ describe('FactRetrieverEngine', () => {
factInsertionAssertionCallback(facts);
return Promise.resolve();
},
insertFactSchema: (ref: string, schema: FactSchema) => {
factSchemaAssertionCallback(ref, schema);
insertFactSchema: (def: FactSchemaDefinition) => {
factSchemaAssertionCallback(def);
return Promise.resolve();
},
} as unknown as TechInsightsStore;
@@ -102,21 +101,19 @@ describe('FactRetrieverEngine', () => {
};
it('Should update fact retriever schemas on initialization', async () => {
factSchemaAssertionCallback = (ref, schema) => {
expect(ref).toEqual('test-factretriever');
factSchemaAssertionCallback = ({ id, schema, version, entityTypes }) => {
expect(id).toEqual('test-factretriever');
expect(version).toEqual('0.0.1');
expect(entityTypes).toEqual(['component']);
expect(schema).toEqual({
version: '0.0.1',
schema: {
testnumberfact: {
type: 'integer',
description: '',
entityKinds: ['component'],
},
testnumberfact: {
type: 'integer',
description: '',
},
});
};
engine = await FactRetrieverEngine.fromConfig(defaultEngineConfig);
engine = await FactRetrieverEngine.create(defaultEngineConfig);
});
it('Should insert facts when scheduled step is run', async () => {
(schedule as jest.Mock).mockImplementation(
@@ -143,7 +140,7 @@ describe('FactRetrieverEngine', () => {
},
});
};
engine = await FactRetrieverEngine.fromConfig(defaultEngineConfig);
engine = await FactRetrieverEngine.create(defaultEngineConfig);
engine.schedule();
const job: any = engine.getJob('test-factretriever');
job.triggerScheduledJobNow();
plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts
+11 -11
View File
@@ -45,7 +45,7 @@ export class FactRetrieverEngine {
private readonly defaultCadence?: string,
) {}
static async fromConfig({
static async create({
repository,
factRetrieverRegistry,
factRetrieverContext,
@@ -59,7 +59,7 @@ export class FactRetrieverEngine {
await Promise.all(
factRetrieverRegistry
.listRetrievers()
.map(it => repository.insertFactSchema(it.ref, it.schema)),
.map(it => repository.insertFactSchema(it)),
);
return new FactRetrieverEngine(
@@ -76,12 +76,12 @@ export class FactRetrieverEngine {
const newRegs: string[] = [];
registrations.forEach(registration => {
const { factRetriever, cadence } = registration;
if (!this.scheduledJobs.has(factRetriever.ref)) {
if (!this.scheduledJobs.has(factRetriever.id)) {
const cronExpression =
cadence || this.defaultCadence || randomDailyCron();
if (!validate(cronExpression)) {
this.logger.warn(
`Validation failed for cron expression ${cronExpression} when trying to schedule fact retriever ${factRetriever.ref}`,
`Validation failed for cron expression ${cronExpression} when trying to schedule fact retriever ${factRetriever.id}`,
);
return;
}
@@ -89,8 +89,8 @@ export class FactRetrieverEngine {
cronExpression,
this.createFactRetrieverHandler(factRetriever),
);
this.scheduledJobs.set(factRetriever.ref, job);
newRegs.push(factRetriever.ref);
this.scheduledJobs.set(factRetriever.id, job);
newRegs.push(factRetriever.id);
}
});
this.logger.info(
@@ -106,27 +106,27 @@ export class FactRetrieverEngine {
return async () => {
const startTimestamp = process.hrtime();
this.logger.info(
`Retrieving facts for fact retriever ${factRetriever.ref}`,
`Retrieving facts for fact retriever ${factRetriever.id}`,
);
const facts = await factRetriever.handler(this.factRetrieverContext);
if (this.logger.isDebugEnabled()) {
this.logger.debug(
`Retrieved ${facts.length} facts for fact retriever ${
factRetriever.ref
factRetriever.id
} in ${duration(startTimestamp)}`,
);
}
try {
await this.repository.insertFacts(factRetriever.ref, facts);
await this.repository.insertFacts(factRetriever.id, facts);
this.logger.info(
`Stored ${facts.length} facts for fact retriever ${
factRetriever.ref
factRetriever.id
} in ${duration(startTimestamp)}`,
);
} catch (e) {
this.logger.warn(
`Failed to insert facts for fact retriever ${factRetriever.ref}`,
`Failed to insert facts for fact retriever ${factRetriever.id}`,
e,
);
}
plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts
+4 -4
View File
@@ -31,19 +31,19 @@ export class FactRetrieverRegistry {
}
register(registration: FactRetrieverRegistration) {
if (this.retrievers.has(registration.factRetriever.ref)) {
if (this.retrievers.has(registration.factRetriever.id)) {
throw new ConflictError(
`Tech insight fact retriever with reference '${registration.factRetriever.ref}' has already been registered`,
`Tech insight fact retriever with identifier '${registration.factRetriever.id}' has already been registered`,
);
}
this.retrievers.set(registration.factRetriever.ref, registration);
this.retrievers.set(registration.factRetriever.id, registration);
}
get(retrieverReference: string): FactRetriever {
const registration = this.retrievers.get(retrieverReference);
if (!registration) {
throw new NotFoundError(
`Tech insight fact retriever with reference '${retrieverReference}' is not registered.`,
`Tech insight fact retriever with identifier '${retrieverReference}' is not registered.`,
);
}
return registration.factRetriever;
plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts
+84 -43
View File
@@ -20,47 +20,58 @@ import { Knex } from 'knex';
const factSchemas = [
{
ref: 'test-schema',
id: 'test-fact',
version: '0.0.1-test',
entityTypes: ['component'],
schema: JSON.stringify({
testNumberFact: {
type: 'integer',
description: 'Test fact with a number type',
entityKinds: ['component'],
},
}),
},
];
const additionalFactSchemas = [
{
ref: 'test-schema',
id: 'test-fact',
version: '1.2.1-test',
entityTypes: ['component'],
schema: JSON.stringify({
testNumberFact: {
type: 'integer',
description: 'Test fact with a number type',
entityKinds: ['component'],
},
testStringFact: {
type: 'string',
description: 'Test fact with a string type',
entityKinds: ['service'],
},
}),
},
{
ref: 'test-schema',
id: 'test-fact',
version: '1.1.1-test',
entityTypes: ['component'],
schema: JSON.stringify({
testStringFact: {
type: 'string',
description: 'Test fact with a string type',
entityKinds: ['service'],
},
}),
},
];
const secondSchema = {
id: 'second-test-fact',
version: '0.0.1-test',
entityTypes: ['service'],
schema: JSON.stringify({
testStringFact: {
type: 'string',
description: 'Test fact with a string type',
},
}),
};
const now = DateTime.now().toISO();
const shortlyInTheFuture = DateTime.now()
.plus(Duration.fromMillis(555))
@@ -72,18 +83,18 @@ const farInTheFuture = DateTime.now()
const facts = [
{
timestamp: now,
ref: 'test-fact',
id: 'test-fact',
version: '0.0.1-test',
entity: 'a/a/a',
entity: 'a:a/a',
facts: JSON.stringify({
testNumberFact: 1,
}),
},
{
timestamp: shortlyInTheFuture,
ref: 'test-fact',
id: 'test-fact',
version: '0.0.1-test',
entity: 'a/a/a',
entity: 'a:a/a',
facts: JSON.stringify({
testNumberFact: 2,
}),
@@ -93,9 +104,9 @@ const facts = [
const additionalFacts = [
{
timestamp: farInTheFuture,
ref: 'test-fact',
id: 'test-fact',
version: '0.0.1-test',
entity: 'a/a/a',
entity: 'a:a/a',
facts: JSON.stringify({
testNumberFact: 3,
}),
@@ -114,7 +125,7 @@ describe('Tech Insights database', () => {
});
const baseAssertionFact = {
ref: 'test-fact',
id: 'test-fact',
entity: { namespace: 'a', kind: 'a', name: 'a' },
timestamp: DateTime.fromISO(shortlyInTheFuture),
version: '0.0.1-test',
@@ -124,14 +135,12 @@ describe('Tech Insights database', () => {
it('should be able to return latest schema', async () => {
const schemas = await store.getLatestSchemas();
expect(schemas[0]).toMatchObject({
ref: 'test-schema',
id: 'test-fact',
version: '0.0.1-test',
schema: {
testNumberFact: {
type: 'integer',
description: 'Test fact with a number type',
entityKinds: ['component'],
},
entityTypes: ['component'],
testNumberFact: {
type: 'integer',
description: 'Test fact with a number type',
},
});
});
@@ -141,43 +150,75 @@ describe('Tech Insights database', () => {
const schemas = await store.getLatestSchemas();
expect(schemas[0]).toMatchObject({
ref: 'test-schema',
id: 'test-fact',
version: '1.2.1-test',
schema: {
testNumberFact: {
type: 'integer',
description: 'Test fact with a number type',
entityKinds: ['component'],
},
testStringFact: {
type: 'string',
description: 'Test fact with a string type',
entityKinds: ['service'],
},
entityTypes: ['component'],
testNumberFact: {
type: 'integer',
description: 'Test fact with a number type',
},
testStringFact: {
type: 'string',
description: 'Test fact with a string type',
},
});
});
it('should return latest facts only for the correct ref', async () => {
const returnedFact = await store.getLatestFactsForRefs(
it('should return multiple schemas if those exists', async () => {
await testDbClient.batchInsert('fact_schemas', [
{
...secondSchema,
id: 'second',
},
]);
const schemas = await store.getLatestSchemas();
expect(schemas).toHaveLength(2);
expect(schemas[0]).toMatchObject({
id: 'test-fact',
version: '1.2.1-test',
entityTypes: ['component'],
testNumberFact: {
type: 'integer',
description: 'Test fact with a number type',
},
testStringFact: {
type: 'string',
description: 'Test fact with a string type',
},
});
expect(schemas[1]).toMatchObject({
id: 'second',
version: '0.0.1-test',
entityTypes: ['service'],
testStringFact: {
type: 'string',
description: 'Test fact with a string type',
},
});
});
it('should return latest facts only for the correct id', async () => {
const returnedFact = await store.getLatestFactsByIds(
['test-fact'],
'a/a/a',
'a:a/a',
);
expect(returnedFact['test-fact']).toMatchObject(baseAssertionFact);
});
it('should return latest facts for multiple refs', async () => {
it('should return latest facts for multiple ids', async () => {
await testDbClient.batchInsert('fact_schemas', [secondSchema]);
await testDbClient.batchInsert(
'facts',
additionalFacts.map(fact => ({
...fact,
ref: 'second-test-fact',
id: 'second-test-fact',
timestamp: farInTheFuture,
})),
);
const returnedFacts = await store.getLatestFactsForRefs(
const returnedFacts = await store.getLatestFactsByIds(
['test-fact', 'second-test-fact'],
'a/a/a',
'a:a/a',
);
expect(returnedFacts['test-fact']).toMatchObject({
@@ -185,7 +226,7 @@ describe('Tech Insights database', () => {
});
expect(returnedFacts['second-test-fact']).toMatchObject({
...baseAssertionFact,
ref: 'second-test-fact',
id: 'second-test-fact',
timestamp: DateTime.fromISO(farInTheFuture),
facts: { testNumberFact: 3 },
});
@@ -193,9 +234,9 @@ describe('Tech Insights database', () => {
it('should return facts correctly between time range', async () => {
await testDbClient.batchInsert('facts', additionalFacts);
const returnedFacts = await store.getFactsBetweenTimestampsForRefs(
const returnedFacts = await store.getFactsBetweenTimestampsByIds(
['test-fact'],
'a/a/a',
'a:a/a',
DateTime.fromISO(now),
DateTime.fromISO(shortlyInTheFuture).plus(Duration.fromMillis(10)),
);
plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts
+48 -46
View File
@@ -19,14 +19,16 @@ import {
TechInsightFact,
FlatTechInsightFact,
TechInsightsStore,
FactSchemaDefinition,
} from '@backstage/plugin-tech-insights-common';
import { rsort } from 'semver';
import { groupBy } from 'lodash';
import { groupBy, omit } from 'lodash';
import { DateTime } from 'luxon';
import { Logger } from 'winston';
import { parseEntityName, stringifyEntityRef } from '@backstage/catalog-model';
export type RawDbFactRow = {
ref: string;
id: string;
version: string;
timestamp: Date | string;
entity: string;
@@ -34,10 +36,10 @@ export type RawDbFactRow = {
};
type RawDbFactSchemaRow = {
id: number;
ref: string;
id: string;
version: string;
schema: string;
entityTypes?: string;
};
export class TechInsightsDatabase implements TechInsightsStore {
@@ -45,51 +47,51 @@ export class TechInsightsDatabase implements TechInsightsStore {
constructor(private readonly db: Knex, private readonly logger: Logger) {}
async getLatestSchemas(refs?: string[]): Promise<FactSchema[]> {
async getLatestSchemas(ids?: string[]): Promise<FactSchema[]> {
const queryBuilder = this.db<RawDbFactSchemaRow>('fact_schemas');
if (refs) {
queryBuilder.whereIn('ref', refs);
if (ids) {
queryBuilder.whereIn('id', ids);
}
const existingSchemas = await queryBuilder.orderBy('id', 'desc').select();
const groupedSchemas = groupBy(existingSchemas, 'ref');
const groupedSchemas = groupBy(existingSchemas, 'id');
return Object.values(groupedSchemas)
.map(schemas => {
const sorted = rsort(schemas.map(it => it.version));
return schemas.find(it => it.version === sorted[0])!!;
})
.map((it: RawDbFactSchemaRow) => ({
...it,
schema: JSON.parse(it.schema),
...omit(it, 'schema'),
...JSON.parse(it.schema),
entityTypes: it.entityTypes ? it.entityTypes.split(',') : [],
}));
}
async insertFactSchema(ref: string, schema: FactSchema) {
async insertFactSchema(schemaDefinition: FactSchemaDefinition) {
const { id, version, schema, entityTypes } = schemaDefinition;
const existingSchemas = await this.db<RawDbFactSchemaRow>('fact_schemas')
.where({ ref })
.where({ id })
.and.where({ version })
.select();
const exists = existingSchemas.some(
it => it.ref === ref && it.version === schema.version,
);
if (!exists) {
if (!existingSchemas || existingSchemas.length === 0) {
await this.db<RawDbFactSchemaRow>('fact_schemas').insert({
ref,
version: schema.version,
schema: JSON.stringify(schema.schema),
id,
version,
entityTypes: entityTypes && entityTypes.join(','),
schema: JSON.stringify(schema),
});
}
}
async insertFacts(ref: string, facts: TechInsightFact[]): Promise<void> {
async insertFacts(id: string, facts: TechInsightFact[]): Promise<void> {
if (facts.length === 0) return;
const currentSchema = await this.getLatestSchema(ref);
const currentSchema = await this.getLatestSchema(id);
const factRows = facts.map(it => {
const { namespace, name, kind } = it.entity;
return {
ref: ref,
id,
version: currentSchema.version,
entity: `${namespace}/${kind}/${name}`.toLocaleLowerCase('en-US'),
entity: stringifyEntityRef(it.entity),
facts: JSON.stringify(it.facts),
...(it.timestamp && { timestamp: it.timestamp.toJSDate() }),
};
@@ -99,36 +101,36 @@ export class TechInsightsDatabase implements TechInsightsStore {
});
}
async getLatestFactsForRefs(
refs: string[],
async getLatestFactsByIds(
ids: string[],
entityTriplet: string,
): Promise<{ [p: string]: FlatTechInsightFact }> {
): Promise<{ [factId: string]: FlatTechInsightFact }> {
const results = await this.db<RawDbFactRow>('facts')
.where({ entity: entityTriplet })
.and.whereIn('ref', refs)
.and.whereIn('id', ids)
.join(
this.db('facts')
.max('timestamp')
.column('ref as subRef')
.groupBy('ref')
.column('id as subId')
.groupBy('id')
.as('subQ'),
'facts.ref',
'subQ.subRef',
'facts.id',
'subQ.subId',
);
return this.dbFactRowsToTechInsightFacts(results);
}
async getFactsBetweenTimestampsForRefs(
refs: string[],
async getFactsBetweenTimestampsByIds(
ids: string[],
entityTriplet: string,
startDateTime: DateTime,
endDateTime: DateTime,
): Promise<{
[p: string]: FlatTechInsightFact[];
[factId: string]: FlatTechInsightFact[];
}> {
const results = await this.db<RawDbFactRow>('facts')
.where({ entity: entityTriplet })
.and.whereIn('ref', refs)
.and.whereIn('id', ids)
.and.whereBetween('timestamp', [
startDateTime.toISO(),
endDateTime.toISO(),
@@ -136,31 +138,31 @@ export class TechInsightsDatabase implements TechInsightsStore {
return groupBy(
results.map(it => {
const [namespace, kind, name] = it.entity.split('/');
const { namespace, kind, name } = parseEntityName(it.entity);
const timestamp =
typeof it.timestamp === 'string'
? DateTime.fromISO(it.timestamp)
: DateTime.fromJSDate(it.timestamp);
return {
ref: it.ref,
id: it.id,
entity: { namespace, kind, name },
timestamp,
version: it.version,
facts: JSON.parse(it.facts),
};
}),
'ref',
'id',
);
}
private async getLatestSchema(ref: string): Promise<RawDbFactSchemaRow> {
private async getLatestSchema(id: string): Promise<RawDbFactSchemaRow> {
const existingSchemas = await this.db<RawDbFactSchemaRow>('fact_schemas')
.where({ ref })
.where({ id })
.orderBy('id', 'desc')
.select();
if (existingSchemas.length < 1) {
this.logger.warn(`No schema found for ${ref}. `);
throw new Error(`No schema found for ${ref}. `);
this.logger.warn(`No schema found for ${id}. `);
throw new Error(`No schema found for ${id}. `);
}
const sorted = rsort(existingSchemas.map(it => it.version));
return existingSchemas.find(it => it.version === sorted[0])!!;
@@ -168,15 +170,15 @@ export class TechInsightsDatabase implements TechInsightsStore {
private dbFactRowsToTechInsightFacts(rows: RawDbFactRow[]) {
return rows.reduce((acc, it) => {
const [namespace, kind, name] = it.entity.split('/');
const { namespace, kind, name } = parseEntityName(it.entity);
const timestamp =
typeof it.timestamp === 'string'
? DateTime.fromISO(it.timestamp)
: DateTime.fromJSDate(it.timestamp);
return {
...acc,
[it.ref]: {
ref: it.ref,
[it.id]: {
id: it.id,
entity: { namespace, kind, name },
timestamp,
version: it.version,
plugins/tech-insights-backend/src/service/router.test.ts
+28 -23
View File
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { DefaultTechInsightsBuilder } from './DefaultTechInsightsBuilder';
import { buildTechInsightsContext } from './techInsightsContextBuilder';
import { createRouter } from './router';
import { getVoidLogger } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
@@ -27,14 +27,14 @@ import { Knex } from 'knex';
describe('Tech Insights router tests', () => {
let app: express.Express;
const latestFactsForRefsMock = jest.fn();
const factsBetweenTimestampsForRefsMock = jest.fn();
const latestFactsByIdsMock = jest.fn();
const factsBetweenTimestampsByIdsMock = jest.fn();
const latestSchemasMock = jest.fn();
const mockPersistenceContext: PersistenceContext = {
techInsightsStore: {
getLatestFactsForRefs: latestFactsForRefsMock,
getFactsBetweenTimestampsForRefs: factsBetweenTimestampsForRefsMock,
getLatestFactsByIds: latestFactsByIdsMock,
getFactsBetweenTimestampsByIds: factsBetweenTimestampsByIdsMock,
getLatestSchemas: latestSchemasMock,
} as unknown as TechInsightsStore,
};
@@ -44,7 +44,7 @@ describe('Tech Insights router tests', () => {
});
beforeAll(async () => {
const techInsightsContext = await new DefaultTechInsightsBuilder({
const techInsightsContext = await buildTechInsightsContext({
database: {
getClient: () => {
return Promise.resolve({
@@ -61,7 +61,7 @@ describe('Tech Insights router tests', () => {
getBaseUrl: (_: string) => Promise.resolve('http://mock.url'),
getExternalBaseUrl: (_: string) => Promise.resolve('http://mock.url'),
},
}).build();
});
const router = await createRouter({
logger: getVoidLogger(),
@@ -80,32 +80,36 @@ describe('Tech Insights router tests', () => {
it('should not contain check endpoints when checker not present', async () => {
await request(app).get('/checks').expect(404);
await request(app).get('/checks/a/a/a').expect(404);
await request(app).post('/checks/a/a/a').expect(404);
});
it('should parse be able to parse ref request params for fact retrieval', async () => {
it('should be able to parse id request params for fact retrieval', async () => {
await request(app)
.get('/facts/latest/a/a/a')
.query({ refs: ['firstref', 'secondref'] })
.get('/facts/latest')
.query({
entity: 'a:a/a',
ids: ['firstId', 'secondId'],
})
.expect(200);
expect(latestFactsForRefsMock).toHaveBeenCalledWith(
['firstref', 'secondref'],
'a/a/a',
expect(latestFactsByIdsMock).toHaveBeenCalledWith(
['firstId', 'secondId'],
'a:a/a',
);
});
it('should parse be able to parse datetime request params for fact retrieval', async () => {
it('should be able to parse datetime request params for fact retrieval', async () => {
await request(app)
.get('/facts/range/a/a/a')
.get('/facts/range')
.query({
refs: ['firstref', 'secondref'],
entity: 'a:a/a',
ids: ['firstId', 'secondId'],
startDatetime: '2021-12-12T12:12:12',
endDatetime: '2022-11-11T11:11:11',
})
.expect(200);
expect(factsBetweenTimestampsForRefsMock).toHaveBeenCalledWith(
['firstref', 'secondref'],
'a/a/a',
expect(factsBetweenTimestampsByIdsMock).toHaveBeenCalledWith(
['firstId', 'secondId'],
'a:a/a',
DateTime.fromISO('2021-12-12T12:12:12.000+00:00'),
DateTime.fromISO('2022-11-11T11:11:11.000+00:00'),
);
@@ -113,13 +117,14 @@ describe('Tech Insights router tests', () => {
it('should respond gracefully on parsing errors', async () => {
await request(app)
.get('/facts/range/a/a/a')
.get('/facts/range')
.query({
refs: ['firstref', 'secondref'],
entity: 'a:a/a',
ids: ['firstId', 'secondId'],
startDatetime: '2021-12-1222T12:12:12',
endDatetime: '2022-1122-11T11:11:11',
})
.expect(422);
expect(latestFactsForRefsMock).toHaveBeenCalledTimes(0);
expect(latestFactsByIdsMock).toHaveBeenCalledTimes(0);
});
});
plugins/tech-insights-backend/src/service/router.ts
+30 -14
View File
@@ -25,6 +25,11 @@ import {
import { Logger } from 'winston';
import { DateTime } from 'luxon';
import { PersistenceContext } from './persistence/DatabaseManager';
import {
EntityRef,
parseEntityName,
stringifyEntityRef,
} from '@backstage/catalog-model';
/**
* @public
@@ -92,7 +97,7 @@ export async function createRouter<
});
}
const { checks }: { checks: string[] } = req.body;
const entityTriplet = `${namespace.toLowerCase()}/${kind.toLowerCase()}/${name.toLowerCase()}`;
const entityTriplet = stringifyEntityRef({ namespace, kind, name });
const checkResult = await factChecker.runChecks(entityTriplet, checks);
return res.send(checkResult);
} catch (e) {
@@ -106,22 +111,33 @@ export async function createRouter<
}
router.get('/fact-schemas', async (req, res) => {
const refs = req.query.refs as string[];
return res.send(await techInsightsStore.getLatestSchemas(refs));
const ids = req.query.ids as string[];
return res.send(await techInsightsStore.getLatestSchemas(ids));
});
router.get('/facts/latest/:namespace/:kind/:name', async (req, res) => {
const { namespace, kind, name } = req.params;
const refs = req.query.refs as string[];
const entityTriplet = `${namespace.toLowerCase()}/${kind.toLowerCase()}/${name.toLowerCase()}`;
/**
* /facts/latest?entity=component:default/mycomponent&ids[]=factRetrieverId1&ids[]=factRetrieverId2
*/
router.get('/facts/latest', async (req, res) => {
const { entity } = req.query;
const { namespace, kind, name } = parseEntityName(entity as EntityRef);
const ids = req.query.ids as string[];
return res.send(
await techInsightsStore.getLatestFactsForRefs(refs, entityTriplet),
await techInsightsStore.getLatestFactsByIds(
ids,
stringifyEntityRef({ namespace, kind, name }),
),
);
});
router.get('/facts/range/:namespace/:kind/:name', async (req, res) => {
const { namespace, kind, name } = req.params;
const refs = req.query.refs as string[];
/**
* /facts/latest?entity=component:default/mycomponent&startDateTime=2021-12-24T01:23:45&endDateTime=2021-12-31T23:59:59&ids[]=factRetrieverId1&ids[]=factRetrieverId2
*/
router.get('/facts/range', async (req, res) => {
const { entity } = req.query;
const { namespace, kind, name } = parseEntityName(entity as EntityRef);
const ids = req.query.ids as string[];
const startDatetime = DateTime.fromISO(req.query.startDatetime as string);
const endDatetime = DateTime.fromISO(req.query.endDatetime as string);
if (!startDatetime.isValid || !endDatetime.isValid) {
@@ -131,10 +147,10 @@ export async function createRouter<
value: !startDatetime.isValid ? startDatetime : endDatetime,
});
}
const entityTriplet = `${namespace.toLowerCase()}/${kind.toLowerCase()}/${name.toLowerCase()}`;
const entityTriplet = stringifyEntityRef({ namespace, kind, name });
return res.send(
await techInsightsStore.getFactsBetweenTimestampsForRefs(
refs,
await techInsightsStore.getFactsBetweenTimestampsByIds(
ids,
entityTriplet,
startDatetime,
endDatetime,
plugins/tech-insights-backend/src/service/DefaultTechInsightsBuilder.ts → plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts
+39 -52
View File
@@ -80,70 +80,57 @@ export type TechInsightsContext<
};
/**
* @public
* @typeParam CheckType - Type of the check for the fact checker this builder returns
* @typeParam CheckResultType - Type of the check result for the fact checker this builder returns
* Constructs needed persistence context, fact retriever engine
* and optionally fact checker implementations to be used in the tech insights module.
*
* Default implementation of TechInsightsBuilder.
* @param options - Needed options to construct TechInsightsContext
* @returns TechInsightsContext with persistence implementations and optionally an implementation of a FactChecker
*/
export class DefaultTechInsightsBuilder<
export const buildTechInsightsContext = async <
CheckType extends TechInsightCheck,
CheckResultType extends CheckResult,
> {
private readonly options: TechInsightsOptions<CheckType, CheckResultType>;
>(
options: TechInsightsOptions<CheckType, CheckResultType>,
): Promise<TechInsightsContext<CheckType, CheckResultType>> => {
const {
factRetrievers,
factCheckerFactory,
config,
discovery,
database,
logger,
} = options;
constructor(options: TechInsightsOptions<CheckType, CheckResultType>) {
this.options = options;
}
const factRetrieverRegistry = new FactRetrieverRegistry(factRetrievers);
/**
* Constructs needed persistence context, fact retriever engine
* and optionally fact checker implementations to be used in the tech insights module.
*
* @returns TechInsightsContext with persistence implementations and optionally an implementation of a FactChecker
*/
async build(): Promise<TechInsightsContext<CheckType, CheckResultType>> {
const {
factRetrievers,
factCheckerFactory,
const persistenceContext = await DatabaseManager.initializePersistenceContext(
await database.getClient(),
{ logger },
);
const factRetrieverEngine = await FactRetrieverEngine.create({
repository: persistenceContext.techInsightsStore,
factRetrieverRegistry,
factRetrieverContext: {
config,
discovery,
database,
logger,
} = this.options;
},
});
const factRetrieverRegistry = new FactRetrieverRegistry(factRetrievers);
const persistenceContext =
await DatabaseManager.initializePersistenceContext(
await database.getClient(),
{ logger },
);
const factRetrieverEngine = await FactRetrieverEngine.fromConfig({
repository: persistenceContext.techInsightsStore,
factRetrieverRegistry,
factRetrieverContext: {
config,
discovery,
logger,
},
});
factRetrieverEngine.schedule();
if (factCheckerFactory) {
const factChecker = factCheckerFactory.construct(
persistenceContext.techInsightsStore,
);
return {
persistenceContext,
factChecker,
};
}
factRetrieverEngine.schedule();
if (factCheckerFactory) {
const factChecker = factCheckerFactory.construct(
persistenceContext.techInsightsStore,
);
return {
persistenceContext,
factChecker,
};
}
}
return {
persistenceContext,
};
};