diff --git a/packages/backend/src/plugins/techInsights.ts b/packages/backend/src/plugins/techInsights.ts index 6e10b60370..abab1d133d 100644 --- a/packages/backend/src/plugins/techInsights.ts +++ b/packages/backend/src/plugins/techInsights.ts @@ -39,6 +39,7 @@ export default async function createPlugin({ discovery, factRetrievers: [ createFactRetrieverRegistration('5 4 * * 6', { + // Example cron, At 04:05 on Saturday. id: 'testRetriever', version: '1.1.2', entityFilter: [{ kind: 'component' }], // EntityFilter to be used in the future (creating checks, graphs etc.) to figure out which entities this fact retrieves data for. @@ -52,7 +53,9 @@ export default async function createPlugin({ const catalogClient = new CatalogClient({ discoveryApi: discovery, }); - const entities = await catalogClient.getEntities(); + const entities = await catalogClient.getEntities({ + filter: [{ kind: 'component' }], + }); return Promise.resolve( entities.items.map(it => { diff --git a/plugins/tech-insights-backend-module-jsonfc/README.md b/plugins/tech-insights-backend-module-jsonfc/README.md index 1404bd4ea4..a19c3dd2ce 100644 --- a/plugins/tech-insights-backend-module-jsonfc/README.md +++ b/plugins/tech-insights-backend-module-jsonfc/README.md @@ -24,31 +24,31 @@ and modify the `techInsights.ts` file to contain a reference to the FactCheckers + logger, + }), -const builder = new DefaultTechInsightsBuilder({ -logger, -config, -database, -discovery, -factRetrievers: [myFactRetrieverRegistration], -+ factCheckerFactory: myFactCheckerFactory -}); + const builder = new DefaultTechInsightsBuilder({ + logger, + config, + database, + discovery, + factRetrievers: [myFactRetrieverRegistration], ++ factCheckerFactory: myFactCheckerFactory + }); ``` By default this implementation comes with an in-memory storage to store checks. You can inject an additional data store by adding an implementation of `TechInsightCheckRegistry` into the constructor options when creating a `JsonRulesEngineFactCheckerFactory`. That can be done as follows ```diff -const myTechInsightCheckRegistry: TechInsightCheckRegistry = // snip -const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ - checks: [], - logger, -+ checkRegistry: myTechInsightCheckRegistry -}), + const myTechInsightCheckRegistry: TechInsightCheckRegistry = // snip + const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ + checks: [], + logger, ++ checkRegistry: myTechInsightCheckRegistry + }), ``` ## Adding checks -Checks for this FactChecker are constructed as `json-rules-engine` compatible JSON rules. A check could look like the following for example: +Checks for this FactChecker are constructed as [`json-rules-engine` compatible JSON rules](https://github.com/CacheControl/json-rules-engine/blob/master/docs/rules.md#conditions). A check could look like the following for example: ```ts import { TechInsightJsonRuleCheck } from '../types'; diff --git a/plugins/tech-insights-backend/README.md b/plugins/tech-insights-backend/README.md index 2de16b65b0..e7a78d34a2 100644 --- a/plugins/tech-insights-backend/README.md +++ b/plugins/tech-insights-backend/README.md @@ -53,20 +53,20 @@ With the `techInsights.ts` router setup in place, add the router to `packages/backend/src/index.ts`: ```diff -+import techInsights from './plugins/techInsights'; ++ import techInsights from './plugins/techInsights'; -async function main() { - ... - const createEnv = makeCreateEnv(config); + async function main() { + ... + const createEnv = makeCreateEnv(config); - const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); -+ const techInsightsEnv = useHotMemoize(module, () => createEnv('tech_insights')); - - const apiRouter = Router(); -+ apiRouter.use('/tech-insights', await techInsights(techInsightsEnv)); - ... - apiRouter.use(notFoundHandler()); + const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); ++ const techInsightsEnv = useHotMemoize(module, () => createEnv('tech_insights')); + const apiRouter = Router(); ++ apiRouter.use('/tech-insights', await techInsights(techInsightsEnv)); + ... + apiRouter.use(notFoundHandler()); + } ``` ### Adding fact retrievers @@ -95,10 +95,10 @@ Then you can modify the example `techInsights.ts` file shown above like this: ```diff const builder = new DefaultTechInsightsBuilder({ -logger, -config, -database, -discovery, + logger, + config, + database, + discovery, - factRetrievers: [], + factRetrievers: [myFactRetrieverRegistration], }); @@ -119,12 +119,12 @@ An example implementation of a FactRetriever could for example be as follows: const myFactRetriever: FactRetriever = { id: 'documentation-number-factretriever', // unique identifier of the fact retriever version: '0.1.1', // SemVer version number of this fact retriever schema. This should be incremented if the implementation changes + entityFilter: [{ kind: 'component' }], // EntityFilter to be used in the future (creating checks, graphs etc.) to figure out which entities this fact retrieves data for. schema: { // Name/identifier of an individual fact that this retriever returns examplenumberfact: { type: 'integer', // Type of the fact description: 'A fact of a number', // Description of the fact - entityTypes: ['component'], // An array of entity kinds that this fact is applicable to }, }, handler: async ctx => { @@ -133,7 +133,9 @@ const myFactRetriever: FactRetriever = { const catalogClient = new CatalogClient({ discoveryApi: discovery, }); - const entities = await catalogClient.getEntities(); // Retrieve all entities + const entities = await catalogClient.getEntities({ + filter: [{ kind: 'component' }], + }); /** * snip: Do complex logic to retrieve facts from external system or calculate fact values */ @@ -183,14 +185,14 @@ and modify the `techInsights.ts` file to contain a reference to the FactChecker + logger, + }), -const builder = new DefaultTechInsightsBuilder({ -logger, -config, -database, -discovery, -factRetrievers: [myFactRetrieverRegistration], -+ factCheckerFactory: myFactCheckerFactory -}); + const builder = new DefaultTechInsightsBuilder({ + logger, + config, + database, + discovery, + factRetrievers: [myFactRetrieverRegistration], ++ factCheckerFactory: myFactCheckerFactory + }); ``` To be able to run checks, you need to additionally add individual checks into your FactChecker implementation. For examples how to add these, you can check the documentation of the individual implementation of the FactChecker diff --git a/plugins/tech-insights-backend/package.json b/plugins/tech-insights-backend/package.json index cd437ee16b..678bec25c4 100644 --- a/plugins/tech-insights-backend/package.json +++ b/plugins/tech-insights-backend/package.json @@ -52,6 +52,7 @@ "yn": "^4.0.0" }, "devDependencies": { + "@backstage/backend-test-utils": "^0.1.8", "@backstage/cli": "^0.8.0", "@types/supertest": "^2.0.8", "@types/node-cron": "^3.0.0", diff --git a/plugins/tech-insights-backend/src/index.ts b/plugins/tech-insights-backend/src/index.ts index 2a74bc6b88..5e8e690e8d 100644 --- a/plugins/tech-insights-backend/src/index.ts +++ b/plugins/tech-insights-backend/src/index.ts @@ -23,5 +23,5 @@ export type { TechInsightsContext, } from './service/techInsightsContextBuilder'; -export type { PersistenceContext } from './service/persistence/DatabaseManager'; +export type { PersistenceContext } from './service/persistence/persistenceContext'; export { createFactRetrieverRegistration } from './service/fact/createFactRetriever'; diff --git a/plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts b/plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts deleted file mode 100644 index 126c253fa5..0000000000 --- a/plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common'; -import knexFactory, { Knex } from 'knex'; -import { Logger } from 'winston'; -import { v4 as uuidv4 } from 'uuid'; -import { TechInsightsDatabase } from './TechInsightsDatabase'; -import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; - -const migrationsDir = resolvePackagePath( - '@backstage/plugin-tech-insights-backend', - 'migrations', -); - -/** - * A Container for persistence related components in TechInsights - * - * @public - */ -export type PersistenceContext = { - techInsightsStore: TechInsightsStore; -}; - -export type CreateDatabaseOptions = { - logger: Logger; -}; - -const defaultOptions: CreateDatabaseOptions = { - logger: getVoidLogger(), -}; - -/** - * A factory class to construct persistence context for both running implmentation and test cases. - * - * @public - */ -export class DatabaseManager { - public static async initializePersistenceContext( - knex: Knex, - options: CreateDatabaseOptions = defaultOptions, - ): Promise { - await knex.migrate.latest({ - directory: migrationsDir, - }); - return { - techInsightsStore: new TechInsightsDatabase(knex, options.logger), - }; - } - - public static async createTestDatabase( - knex: Knex, - ): Promise { - const knexInstance = knex ?? (await this.createTestDatabaseConnection()); - return await this.initializePersistenceContext(knexInstance); - } - - public static async createTestDatabaseConnection(): Promise { - const config: Knex.Config = { - client: 'sqlite3', - connection: ':memory:', - useNullAsDefault: true, - }; - - let knexInstance = knexFactory(config); - if (typeof config.connection !== 'string') { - const tempDbName = `d${uuidv4().replace(/-/g, '')}`; - await knexInstance.raw(`CREATE DATABASE ${tempDbName};`); - knexInstance = knexFactory({ - ...config, - connection: { - ...config.connection, - database: tempDbName, - }, - }); - } - - knexInstance.client.pool.on( - 'createSuccess', - (_eventId: any, resource: any) => { - resource.run('PRAGMA foreign_keys = ON', () => {}); - }, - ); - - return knexInstance; - } -} diff --git a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts index 85d6098ce3..43d00e2425 100644 --- a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts +++ b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts @@ -13,10 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { DatabaseManager } from './DatabaseManager'; import { DateTime, Duration } from 'luxon'; import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; import { Knex } from 'knex'; +import { TestDatabases } from '@backstage/backend-test-utils'; +import { getVoidLogger } from '@backstage/backend-common'; +import { initializePersistenceContext } from './persistenceContext'; const factSchemas = [ { @@ -117,9 +119,14 @@ describe('Tech Insights database', () => { let store: TechInsightsStore; let testDbClient: Knex; beforeAll(async () => { - testDbClient = await DatabaseManager.createTestDatabaseConnection(); - store = (await DatabaseManager.createTestDatabase(testDbClient)) - .techInsightsStore; + testDbClient = await TestDatabases.create().init('SQLITE_3'); + + store = ( + await initializePersistenceContext(testDbClient, { + logger: getVoidLogger(), + }) + ).techInsightsStore; + await testDbClient.batchInsert('fact_schemas', factSchemas); await testDbClient.batchInsert('facts', facts); }); diff --git a/plugins/tech-insights-backend/src/service/persistence/persistenceContext.ts b/plugins/tech-insights-backend/src/service/persistence/persistenceContext.ts new file mode 100644 index 0000000000..b0fb65349f --- /dev/null +++ b/plugins/tech-insights-backend/src/service/persistence/persistenceContext.ts @@ -0,0 +1,59 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common'; +import { Knex } from 'knex'; +import { Logger } from 'winston'; +import { TechInsightsDatabase } from './TechInsightsDatabase'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-tech-insights-backend', + 'migrations', +); + +/** + * A Container for persistence related components in TechInsights + * + * @public + */ +export type PersistenceContext = { + techInsightsStore: TechInsightsStore; +}; + +export type CreateDatabaseOptions = { + logger: Logger; +}; + +const defaultOptions: CreateDatabaseOptions = { + logger: getVoidLogger(), +}; + +/** + * A factory method to construct persistence context for running implementation. + * + * @public + */ +export const initializePersistenceContext = async ( + knex: Knex, + options: CreateDatabaseOptions = defaultOptions, +): Promise => { + await knex.migrate.latest({ + directory: migrationsDir, + }); + return { + techInsightsStore: new TechInsightsDatabase(knex, options.logger), + }; +}; diff --git a/plugins/tech-insights-backend/src/service/router.test.ts b/plugins/tech-insights-backend/src/service/router.test.ts index 8a7b6bdbb3..0b7d3b7c45 100644 --- a/plugins/tech-insights-backend/src/service/router.test.ts +++ b/plugins/tech-insights-backend/src/service/router.test.ts @@ -19,7 +19,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import request from 'supertest'; import express from 'express'; -import { PersistenceContext } from './persistence/DatabaseManager'; +import { PersistenceContext } from './persistence/persistenceContext'; import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; import { DateTime } from 'luxon'; import { Knex } from 'knex'; diff --git a/plugins/tech-insights-backend/src/service/router.ts b/plugins/tech-insights-backend/src/service/router.ts index 752534928f..a8951318ea 100644 --- a/plugins/tech-insights-backend/src/service/router.ts +++ b/plugins/tech-insights-backend/src/service/router.ts @@ -25,7 +25,7 @@ import { import { CheckResult } from '@backstage/plugin-tech-insights-common'; import { Logger } from 'winston'; import { DateTime } from 'luxon'; -import { PersistenceContext } from './persistence/DatabaseManager'; +import { PersistenceContext } from './persistence/persistenceContext'; import { EntityRef, parseEntityName, diff --git a/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts b/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts index 7305f1778c..75e24b7c06 100644 --- a/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts +++ b/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts @@ -29,9 +29,9 @@ import { TechInsightCheck, } from '@backstage/plugin-tech-insights-node'; import { - DatabaseManager, + initializePersistenceContext, PersistenceContext, -} from './persistence/DatabaseManager'; +} from './persistence/persistenceContext'; import { CheckResult } from '@backstage/plugin-tech-insights-common'; /** @@ -105,7 +105,7 @@ export const buildTechInsightsContext = async < const factRetrieverRegistry = new FactRetrieverRegistry(factRetrievers); - const persistenceContext = await DatabaseManager.initializePersistenceContext( + const persistenceContext = await initializePersistenceContext( await database.getClient(), { logger }, ); diff --git a/plugins/tech-insights-node/api-report.md b/plugins/tech-insights-node/api-report.md new file mode 100644 index 0000000000..7a157309dd --- /dev/null +++ b/plugins/tech-insights-node/api-report.md @@ -0,0 +1,155 @@ +## API Report File for "@backstage/plugin-tech-insights-node" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { CheckResult } from '@backstage/plugin-tech-insights-common'; +import { Config } from '@backstage/config'; +import { DateTime } from 'luxon'; +import { Logger as Logger_2 } from 'winston'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; + +// @public +export type CheckValidationResponse = { + valid: boolean; + message?: string; + errors?: unknown[]; +}; + +// @public +export interface FactChecker< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + getChecks(): Promise; + runChecks(entity: string, checks?: string[]): Promise; + validate(check: CheckType): Promise; +} + +// @public +export interface FactCheckerFactory< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + // (undocumented) + construct( + repository: TechInsightsStore, + ): FactChecker; +} + +// @public +export interface FactRetriever { + // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag + // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" + // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag + // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" + entityFilter?: + | Record[] + | Record; + handler: (ctx: FactRetrieverContext) => Promise; + id: string; + schema: FactSchema; + version: string; +} + +// @public +export type FactRetrieverContext = { + config: Config; + discovery: PluginEndpointDiscovery; + logger: Logger_2; +}; + +// @public +export type FactRetrieverRegistration = { + factRetriever: FactRetriever; + cadence?: string; +}; + +// @public +export type FactSchema = { + [name: string]: { + type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; + description: string; + since?: string; + metadata?: Record; + }; +}; + +// Warning: (ae-missing-release-tag) "FactSchemaDefinition" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type FactSchemaDefinition = Omit; + +// @public +export type FlatTechInsightFact = TechInsightFact & { + id: string; +}; + +// @public +export interface TechInsightCheck { + description: string; + factIds: string[]; + failureMetadata?: Record; + id: string; + name: string; + successMetadata?: Record; + type: string; +} + +// @public +export interface TechInsightCheckRegistry { + // (undocumented) + get(checkId: string): Promise; + // (undocumented) + getAll(checks: string[]): Promise; + // (undocumented) + list(): Promise; + // (undocumented) + register(check: CheckType): Promise; +} + +// @public +export type TechInsightFact = { + entity: { + namespace: string; + kind: string; + name: string; + }; + facts: Record< + string, + | number + | string + | boolean + | DateTime + | number[] + | string[] + | boolean[] + | DateTime[] + >; + timestamp?: DateTime; +}; + +// @public +export interface TechInsightsStore { + getFactsBetweenTimestampsByIds( + ids: string[], + entity: string, + startDateTime: DateTime, + endDateTime: DateTime, + ): Promise<{ + [factRef: string]: FlatTechInsightFact[]; + }>; + // (undocumented) + getLatestFactsByIds( + ids: string[], + entity: string, + ): Promise<{ + [factRef: string]: FlatTechInsightFact; + }>; + getLatestSchemas(ids?: string[]): Promise; + insertFacts(id: string, facts: TechInsightFact[]): Promise; + insertFactSchema(schemaDefinition: FactSchemaDefinition): Promise; +} + +// (No @packageDocumentation comment for this package) +```