paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 32 Packages Projects Releases Wiki Activity

updating tasks and tech-insights to work together

Browse Source 
Signed-off-by: Phil Gore <pgore@ea.com>
This commit is contained in:
Phil Gore
2022-02-28 23:16:42 -06:00
committed by Fredrik Adelöw
parent 6bc4253672
commit 231fee736b
18 changed files with 358 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/tech-insights-backend-module-jsonfc/package.json
+1 -2
View File
@@ -46,8 +46,7 @@
"winston": "^3.2.1"
},
"devDependencies": {
"@backstage/cli": "^0.16.1-next.0",
"@types/node-cron": "^3.0.1"
"@backstage/cli": "^0.16.1-next.0"
},
"files": [
"dist"
plugins/tech-insights-backend/api-report.md
+3
View File
@@ -14,6 +14,7 @@ import { FactRetrieverRegistration } from '@backstage/plugin-tech-insights-node'
import { Logger } from 'winston';
import { PluginDatabaseManager } from '@backstage/backend-common';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { PluginTaskScheduler } from '@backstage/backend-tasks';
import { TechInsightCheck } from '@backstage/plugin-tech-insights-node';
import { TechInsightsStore } from '@backstage/plugin-tech-insights-node';
@@ -94,6 +95,8 @@ export interface TechInsightsOptions<
factRetrievers: FactRetrieverRegistration[];
// (undocumented)
logger: Logger;
// (undocumented)
scheduler: PluginTaskScheduler;
}
// (No @packageDocumentation comment for this package)
plugins/tech-insights-backend/package.json
+3 -3
View File
@@ -35,6 +35,7 @@
},
"dependencies": {
"@backstage/backend-common": "^0.13.2-next.0",
"@backstage/backend-tasks": "^0.2.2-next.0",
"@backstage/catalog-client": "^1.0.1-next.0",
"@backstage/catalog-model": "^1.0.1-next.0",
"@backstage/config": "^1.0.0",
@@ -47,7 +48,6 @@
"knex": "^1.0.2",
"lodash": "^4.17.21",
"luxon": "^2.0.2",
"node-cron": "^3.0.0",
"semver": "^7.3.5",
"uuid": "^8.3.2",
"winston": "^3.2.1",
@@ -57,9 +57,9 @@
"@backstage/backend-test-utils": "^0.1.23-next.0",
"@backstage/cli": "^0.16.1-next.0",
"@types/supertest": "^2.0.8",
"@types/node-cron": "^3.0.0",
"@types/semver": "^7.3.8",
"supertest": "^6.1.3"
"supertest": "^6.1.3",
"wait-for-expect": "^3.0.2"
},
"files": [
"dist",
plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts
+153 -89
View File
@@ -22,20 +22,16 @@ import {
} from '@backstage/plugin-tech-insights-node';
import { FactRetrieverRegistry } from './FactRetrieverRegistry';
import { FactRetrieverEngine } from './FactRetrieverEngine';
import { getVoidLogger } from '@backstage/backend-common';
import { DatabaseManager, getVoidLogger } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { schedule } from 'node-cron';
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { TaskScheduler } from '@backstage/backend-tasks';
import waitForExpect from 'wait-for-expect';
jest.mock('node-cron', () => {
const original = jest.requireActual('node-cron');
return {
...original,
schedule: jest.fn(),
};
});
jest.useFakeTimers();
const testFactRetriever: FactRetriever = {
id: 'test-factretriever',
id: 'test_factretriever',
version: '0.0.1',
entityFilter: [{ kind: 'component' }],
schema: {
@@ -59,94 +55,162 @@ const testFactRetriever: FactRetriever = {
];
}),
};
const cadence = '1 * * * *';
const defaultCadence = '1 * * * *';
describe('FactRetrieverEngine', () => {
let engine: FactRetrieverEngine;
let factSchemaAssertionCallback: (
type FactSchemaAssertionCallback = (
factSchemaDefinition: FactSchemaDefinition,
) => void;
let factInsertionAssertionCallback: (facts: TechInsightFact[]) => void;
const mockRepository: TechInsightsStore = {
insertFacts: (facts: TechInsightFact[]) => {
factInsertionAssertionCallback(facts);
return Promise.resolve();
},
insertFactSchema: (def: FactSchemaDefinition) => {
factSchemaAssertionCallback(def);
return Promise.resolve();
},
} as unknown as TechInsightsStore;
jest.setTimeout(15000);
const mockFactRetrieverRegistry: FactRetrieverRegistry = {
listRetrievers(): FactRetriever[] {
return [testFactRetriever];
},
listRegistrations(): FactRetrieverRegistration[] {
return [{ factRetriever: testFactRetriever, cadence }];
},
} as unknown as FactRetrieverRegistry;
type FactInsertionAssertionCallback = ({
facts,
id,
}: {
id: string;
facts: TechInsightFact[];
}) => void;
const defaultEngineConfig = {
factRetrieverContext: {
logger: getVoidLogger(),
config: ConfigReader.fromConfigs([]),
discovery: {
getBaseUrl: (_: string) => Promise.resolve('http://mock.url'),
getExternalBaseUrl: (_: string) => Promise.resolve('http://mock.url'),
function createMockRepository(
insertCallback: FactInsertionAssertionCallback,
assertionCallback: FactSchemaAssertionCallback,
): TechInsightsStore {
return {
async insertFacts(f: { facts: TechInsightFact[]; id: string }) {
insertCallback(f);
},
},
factRetrieverRegistry: mockFactRetrieverRegistry,
repository: mockRepository,
};
it('Should update fact retriever schemas on initialization', async () => {
factSchemaAssertionCallback = ({ id, schema, version, entityFilter }) => {
expect(id).toEqual('test-factretriever');
expect(version).toEqual('0.0.1');
expect(entityFilter).toEqual([{ kind: 'component' }]);
expect(schema).toEqual({
testnumberfact: {
type: 'integer',
description: '',
},
});
};
engine = await FactRetrieverEngine.create(defaultEngineConfig);
});
it('Should insert facts when scheduled step is run', async () => {
(schedule as jest.Mock).mockImplementation(
(cronCadence: string, retrieverAction: Function) => {
return {
cadence: cronCadence,
triggerScheduledJobNow: retrieverAction,
};
async insertFactSchema(def: FactSchemaDefinition) {
assertionCallback(def);
},
);
} as unknown as TechInsightsStore;
}
factSchemaAssertionCallback = () => {};
factInsertionAssertionCallback = facts => {
expect(facts).toHaveLength(1);
expect(facts[0]).toEqual({
ref: 'test-factretriever',
entity: {
namespace: 'a',
kind: 'a',
name: 'a',
},
facts: {
testnumberfact: 1,
},
});
};
engine = await FactRetrieverEngine.create(defaultEngineConfig);
engine.schedule();
const job: any = engine.getJob('test-factretriever');
job.triggerScheduledJobNow();
expect(job.cadence!).toEqual(cadence);
expect(testFactRetriever.handler).toHaveBeenCalledWith(
expect.objectContaining({ entityFilter: testFactRetriever.entityFilter }),
);
function createMockFactRetrieverRegistry(
cadence?: string,
factRetriever?: FactRetriever,
): FactRetrieverRegistry {
const cron = cadence ?? defaultCadence;
const retriever: FactRetriever = factRetriever
? factRetriever
: testFactRetriever;
return {
listRetrievers(): FactRetriever[] {
return [retriever];
},
listRegistrations(): FactRetrieverRegistration[] {
return [{ factRetriever: retriever, cadence: cron }];
},
get: (_: string): FactRetrieverRegistration => {
return { factRetriever: retriever, cadence: cron };
},
} as unknown as FactRetrieverRegistry;
}
const databases = TestDatabases.create({
ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
});
async function createEngine(
databaseId: TestDatabaseId,
insert: FactInsertionAssertionCallback,
schema: FactSchemaAssertionCallback,
cadence?: string,
factRetriever?: FactRetriever,
): Promise<FactRetrieverEngine> {
const knex = await databases.init(databaseId);
const databaseManager: Partial<DatabaseManager> = {
forPlugin: (_: string) => ({
getClient: async () => knex,
}),
};
const manager = databaseManager as DatabaseManager;
const scheduler = new TaskScheduler(manager, getVoidLogger());
return await FactRetrieverEngine.create({
factRetrieverContext: {
logger: getVoidLogger(),
config: ConfigReader.fromConfigs([]),
discovery: {
getBaseUrl: (_: string) => Promise.resolve('http://mock.url'),
getExternalBaseUrl: (_: string) => Promise.resolve('http://mock.url'),
},
},
factRetrieverRegistry: createMockFactRetrieverRegistry(
cadence,
factRetriever,
),
repository: createMockRepository(insert, schema),
scheduler: scheduler.forPlugin('tech-insights'),
});
}
it.each(databases.eachSupportedId())(
'Should update fact retriever schemas on initialization',
async databaseId => {
const schemaAssertionCallback = jest.fn((def: FactSchemaDefinition) => {
expect(def.id).toEqual('test_factretriever');
expect(def.version).toEqual('0.0.1');
expect(def.entityFilter).toEqual([{ kind: 'component' }]);
expect(def.schema).toEqual({
testnumberfact: {
type: 'integer',
description: '',
},
});
});
engine = await createEngine(
databaseId,
() => {},
schemaAssertionCallback,
);
expect(schemaAssertionCallback).toBeCalled();
},
60_000,
);
it.each(databases.eachSupportedId())(
'Should insert facts when scheduled step is run',
async databaseId => {
function insertCallback({
facts,
id,
}: {
id: string;
facts: TechInsightFact[];
}) {
expect(facts).toHaveLength(1);
expect(id).toEqual('test_factretriever');
expect(facts[0]).toEqual({
entity: {
namespace: 'a',
kind: 'a',
name: 'a',
},
facts: {
testnumberfact: 1,
},
});
}
engine = await createEngine(databaseId, insertCallback, () => {});
engine.schedule();
const job: FactRetrieverRegistration = engine.getJobRegistration(
testFactRetriever.id,
);
expect(job.cadence!!).toEqual(defaultCadence);
await engine.triggerJob(job.factRetriever.id);
jest.advanceTimersByTime(5000);
await waitForExpect(() => {
expect(testFactRetriever.handler).toHaveBeenCalledWith(
expect.objectContaining({
entityFilter: testFactRetriever.entityFilter,
}),
);
});
},
60_000,
);
});
plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts
+35 -22
View File
@@ -17,12 +17,14 @@ import {
FactLifecycle,
FactRetriever,
FactRetrieverContext,
FactRetrieverRegistration,
TechInsightFact,
TechInsightsStore,
} from '@backstage/plugin-tech-insights-node';
import { FactRetrieverRegistry } from './FactRetrieverRegistry';
import { schedule, validate, ScheduledTask } from 'node-cron';
import { Logger } from 'winston';
import { PluginTaskScheduler } from '@backstage/backend-tasks';
import { Duration } from 'luxon';
function randomDailyCron() {
const rand = (min: number, max: number) =>
@@ -37,26 +39,30 @@ function duration(startTimestamp: [number, number]): string {
}
export class FactRetrieverEngine {
private scheduledJobs = new Map<string, ScheduledTask>();
constructor(
private readonly repository: TechInsightsStore,
private readonly factRetrieverRegistry: FactRetrieverRegistry,
private readonly factRetrieverContext: FactRetrieverContext,
private readonly logger: Logger,
private readonly scheduler: PluginTaskScheduler,
private readonly defaultCadence?: string,
private readonly defaultTimeout?: Duration,
) {}
static async create({
repository,
factRetrieverRegistry,
factRetrieverContext,
scheduler,
defaultCadence,
defaultTimeout,
}: {
repository: TechInsightsStore;
factRetrieverRegistry: FactRetrieverRegistry;
factRetrieverContext: FactRetrieverContext;
scheduler: PluginTaskScheduler;
defaultCadence?: string;
defaultTimeout?: Duration;
}) {
await Promise.all(
factRetrieverRegistry
@@ -69,39 +75,46 @@ export class FactRetrieverEngine {
factRetrieverRegistry,
factRetrieverContext,
factRetrieverContext.logger,
scheduler,
defaultCadence,
defaultTimeout,
);
}
schedule() {
const registrations = this.factRetrieverRegistry.listRegistrations();
const newRegs: string[] = [];
registrations.forEach(registration => {
const { factRetriever, cadence, lifecycle } = registration;
if (!this.scheduledJobs.has(factRetriever.id)) {
const cronExpression =
cadence || this.defaultCadence || randomDailyCron();
if (!validate(cronExpression)) {
this.logger.warn(
`Validation failed for cron expression ${cronExpression} when trying to schedule fact retriever ${factRetriever.id}`,
);
return;
}
const job = schedule(
cronExpression,
this.createFactRetrieverHandler(factRetriever, lifecycle),
registrations.forEach(async registration => {
const { factRetriever, cadence, lifecycle, timeout } = registration;
const cronExpression =
cadence || this.defaultCadence || randomDailyCron();
const timeLimit =
timeout || this.defaultTimeout || Duration.fromObject({ minutes: 5 });
try {
await this.scheduler.scheduleTask({
id: factRetriever.id,
frequency: { cron: cronExpression },
fn: this.createFactRetrieverHandler(factRetriever, lifecycle),
timeout: timeLimit,
});
} catch (e) {
throw new Error(
`Failed to schedule fact retriever ${factRetriever.id}, ${e}`,
);
this.scheduledJobs.set(factRetriever.id, job);
newRegs.push(factRetriever.id);
}
newRegs.push(factRetriever.id);
});
this.logger.info(
`Scheduled ${newRegs.length} fact retrievers to Fact Retriever Engine.`,
`Scheduled ${newRegs.length} fact retrievers to Fact Retriever Engine through Backend Tasks`,
);
}
getJob(ref: string) {
return this.scheduledJobs.get(ref);
getJobRegistration(ref: string): FactRetrieverRegistration {
return this.factRetrieverRegistry.get(ref);
}
async triggerJob(ref: string): Promise<void> {
return this.scheduler.triggerTask(ref);
}
private createFactRetrieverHandler(
plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts
+2 -2
View File
@@ -39,14 +39,14 @@ export class FactRetrieverRegistry {
this.retrievers.set(registration.factRetriever.id, registration);
}
get(retrieverReference: string): FactRetriever {
get(retrieverReference: string): FactRetrieverRegistration {
const registration = this.retrievers.get(retrieverReference);
if (!registration) {
throw new NotFoundError(
`Tech insight fact retriever with identifier '${retrieverReference}' is not registered.`,
);
}
return registration.factRetriever;
return registration;
}
listRetrievers(): FactRetriever[] {
plugins/tech-insights-backend/src/service/router.test.ts
+17 -1
View File
@@ -15,7 +15,7 @@
*/
import { buildTechInsightsContext } from './techInsightsContextBuilder';
import { createRouter } from './router';
import { getVoidLogger } from '@backstage/backend-common';
import { DatabaseManager, getVoidLogger } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import request from 'supertest';
import express from 'express';
@@ -23,6 +23,7 @@ import { PersistenceContext } from './persistence/persistenceContext';
import { TechInsightsStore } from '@backstage/plugin-tech-insights-node';
import { DateTime } from 'luxon';
import { Knex } from 'knex';
import { TaskScheduler } from '@backstage/backend-tasks';
describe('Tech Insights router tests', () => {
let app: express.Express;
@@ -44,6 +45,18 @@ describe('Tech Insights router tests', () => {
});
beforeAll(async () => {
const databaseManager: Partial<DatabaseManager> = {
forPlugin: () => ({
getClient: () => {
return Promise.resolve({
migrate: {
latest: () => {},
},
}) as unknown as Promise<Knex>;
},
}),
};
const manager = databaseManager as DatabaseManager;
const techInsightsContext = await buildTechInsightsContext({
database: {
getClient: () => {
@@ -56,6 +69,9 @@ describe('Tech Insights router tests', () => {
},
logger: getVoidLogger(),
factRetrievers: [],
scheduler: new TaskScheduler(manager, getVoidLogger()).forPlugin(
'tech-insights',
),
config: ConfigReader.fromConfigs([]),
discovery: {
getBaseUrl: (_: string) => Promise.resolve('http://mock.url'),
plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts
+4
View File
@@ -33,6 +33,7 @@ import {
PersistenceContext,
} from './persistence/persistenceContext';
import { CheckResult } from '@backstage/plugin-tech-insights-common';
import { PluginTaskScheduler } from '@backstage/backend-tasks';
/**
* @public
@@ -61,6 +62,7 @@ export interface TechInsightsOptions<
config: Config;
discovery: PluginEndpointDiscovery;
database: PluginDatabaseManager;
scheduler: PluginTaskScheduler;
}
/**
@@ -101,6 +103,7 @@ export const buildTechInsightsContext = async <
discovery,
database,
logger,
scheduler,
} = options;
const factRetrieverRegistry = new FactRetrieverRegistry(factRetrievers);
@@ -111,6 +114,7 @@ export const buildTechInsightsContext = async <
);
const factRetrieverEngine = await FactRetrieverEngine.create({
scheduler,
repository: persistenceContext.techInsightsStore,
factRetrieverRegistry,
factRetrieverContext: {
plugins/tech-insights-node/api-report.md
+2
View File
@@ -6,6 +6,7 @@
import { CheckResult } from '@backstage/plugin-tech-insights-common';
import { Config } from '@backstage/config';
import { DateTime } from 'luxon';
import { Duration } from 'luxon';
import { DurationLike } from 'luxon';
import { Logger } from 'winston';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
@@ -66,6 +67,7 @@ export type FactRetrieverContext = {
export type FactRetrieverRegistration = {
factRetriever: FactRetriever;
cadence?: string;
timeout?: Duration;
lifecycle?: FactLifecycle;
};
plugins/tech-insights-node/src/facts.ts
+8 -2
View File
@@ -13,8 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { DateTime, DurationLike } from 'luxon';
import { DateTime, Duration, DurationLike } from 'luxon';
import { Config } from '@backstage/config';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { Logger } from 'winston';
@@ -243,6 +242,13 @@ export type FactRetrieverRegistration = {
*/
cadence?: string;
/**
* A duration to determine how long the fact retriever should be allowed to run,
* defaults to 5 minutes.
*
*/
timeout?: Duration;
/**
* Fact lifecycle definition
*