From 9990df8a1f5811e2c0f1163b685cfa1f2d46ba63 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 14 Oct 2021 17:38:51 +0100 Subject: [PATCH 01/58] Exports required to implement scaffolder broker Signed-off-by: Brian Fletcher --- .changeset/lovely-cars-sneeze.md | 5 + plugins/scaffolder-backend/api-report.md | 309 ++++++++++++++++++ .../src/scaffolder/index.ts | 24 ++ .../src/scaffolder/tasks/DatabaseTaskStore.ts | 8 + .../scaffolder/tasks/DefaultWorkflowRunner.ts | 4 + .../scaffolder/tasks/LegacyWorkflowRunner.ts | 4 + .../src/scaffolder/tasks/StorageTaskBroker.ts | 11 +- .../src/scaffolder/tasks/TaskWorker.ts | 4 + .../src/scaffolder/tasks/index.ts | 4 +- .../src/scaffolder/tasks/types.ts | 65 +++- .../scaffolder-backend/src/service/router.ts | 16 +- 11 files changed, 445 insertions(+), 9 deletions(-) create mode 100644 .changeset/lovely-cars-sneeze.md diff --git a/.changeset/lovely-cars-sneeze.md b/.changeset/lovely-cars-sneeze.md new file mode 100644 index 0000000000..8e6bfd691c --- /dev/null +++ b/.changeset/lovely-cars-sneeze.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Expose some classes and interfaces public so TaskWorkers can run externally from the scaffolder API. diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index ef4defcc50..e0e7ad7ee5 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -16,6 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import express from 'express'; import { JsonObject } from '@backstage/config'; import { JsonValue } from '@backstage/config'; +import { Knex } from 'knex'; import { LocationSpec } from '@backstage/catalog-model'; import { Logger as Logger_2 } from 'winston'; import { Octokit } from '@octokit/rest'; @@ -25,6 +26,7 @@ import { ScmIntegrationRegistry } from '@backstage/integration'; import { ScmIntegrations } from '@backstage/integration'; import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; import { UrlReader } from '@backstage/backend-common'; +import * as winston from 'winston'; import { Writable } from 'stream'; // Warning: (ae-forgotten-export) The symbol "InputBase" needs to be exported by the entry point index.d.ts @@ -189,6 +191,78 @@ export const createTemplateAction: < templateAction: TemplateAction, ) => TemplateAction; +// @public +export class DatabaseTaskStore implements TaskStore { + constructor(db: Knex); + // (undocumented) + claimTask(): Promise; + // (undocumented) + completeTask({ + taskId, + status, + eventBody, + }: { + taskId: string; + status: Status; + eventBody: JsonObject; + }): Promise; + // (undocumented) + static create(knex: Knex): Promise; + // (undocumented) + createTask( + spec: TaskSpec, + secrets?: TaskSecrets, + ): Promise<{ + taskId: string; + }>; + // (undocumented) + emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise; + // (undocumented) + getTask(taskId: string): Promise; + // (undocumented) + heartbeatTask(taskId: string): Promise; + // Warning: (ae-forgotten-export) The symbol "TaskStoreGetEventsOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + listEvents({ taskId, after }: TaskStoreGetEventsOptions): Promise<{ + events: DbTaskEventRow[]; + }>; + // (undocumented) + listStaleTasks({ timeoutS }: { timeoutS: number }): Promise<{ + tasks: { + taskId: string; + }[]; + }>; +} + +// @public +export type DbTaskRow = { + id: string; + spec: TaskSpec; + status: Status; + createdAt: string; + lastHeartbeatAt?: string; + secrets?: TaskSecrets; +}; + +// Warning: (ae-forgotten-export) The symbol "WorkflowRunner" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "DefaultWorkflowRunner" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class DefaultWorkflowRunner implements WorkflowRunner { + // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts + constructor(options: Options_3); + // Warning: (ae-forgotten-export) The symbol "WorkflowResponse" needs to be exported by the entry point index.d.ts + // + // (undocumented) + execute(task: Task): Promise; +} + +// @public +export type DispatchResult = { + taskId: string; +}; + // Warning: (ae-missing-release-tag) "fetchContents" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -206,6 +280,14 @@ export function fetchContents({ outputPath: string; }): Promise; +// @public +export class LegacyWorkflowRunner implements WorkflowRunner { + // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts + constructor(options: Options_2); + // (undocumented) + execute(task: Task): Promise; +} + // Warning: (ae-missing-release-tag) "OctokitProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -216,6 +298,15 @@ export class OctokitProvider { getOctokit(repoUrl: string): Promise; } +// @public +export type RawDbTaskEventRow = { + id: number; + task_id: string; + body: string; + event_type: TaskEventType; + created_at: string; +}; + // Warning: (ae-missing-release-tag) "RouterOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -235,6 +326,8 @@ export interface RouterOptions { // (undocumented) reader: UrlReader; // (undocumented) + taskBroker?: TaskBroker; + // (undocumented) taskWorkers?: number; } @@ -260,6 +353,218 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { validateEntityKind(entity: Entity): Promise; } +// @public +export type Status = + | 'open' + | 'processing' + | 'failed' + | 'cancelled' + | 'completed'; + +// @public +export class StorageTaskBroker implements TaskBroker { + constructor(storage: TaskStore, logger: Logger_2); + // (undocumented) + claim(): Promise; + // (undocumented) + dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; + // (undocumented) + get(taskId: string): Promise; + // (undocumented) + protected readonly logger: Logger_2; + // (undocumented) + observe( + options: { + taskId: string; + after: number | undefined; + }, + callback: ( + error: Error | undefined, + result: { + events: DbTaskEventRow[]; + }, + ) => void, + ): () => void; + // (undocumented) + protected readonly storage: TaskStore; + // (undocumented) + vacuumTasks(timeoutS: { timeoutS: number }): Promise; +} + +// @public +export interface Task { + // Warning: (ae-forgotten-export) The symbol "CompletedTaskState" needs to be exported by the entry point index.d.ts + // + // (undocumented) + complete(result: CompletedTaskState, metadata?: JsonValue): Promise; + // (undocumented) + done: boolean; + // (undocumented) + emitLog(message: string, metadata?: JsonValue): Promise; + // (undocumented) + getWorkspaceName(): Promise; + // (undocumented) + secrets?: TaskSecrets; + // (undocumented) + spec: TaskSpec; +} + +// Warning: (ae-missing-release-tag) "TaskAgent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class TaskAgent implements Task { + // (undocumented) + complete(result: CompletedTaskState, metadata?: JsonObject): Promise; + // Warning: (ae-forgotten-export) The symbol "TaskState" needs to be exported by the entry point index.d.ts + // + // (undocumented) + static create( + state: TaskState, + storage: TaskStore, + logger: Logger_2, + ): TaskAgent; + // (undocumented) + get done(): boolean; + // (undocumented) + emitLog(message: string, metadata?: JsonObject): Promise; + // (undocumented) + getWorkspaceName(): Promise; + // (undocumented) + get secrets(): TaskSecrets | undefined; + // (undocumented) + get spec(): TaskSpec; +} + +// @public +export interface TaskBroker { + // (undocumented) + claim(): Promise; + // (undocumented) + dispatch( + spec: TaskSpec, + secretTaskSecretss?: TaskSecrets, + ): Promise; + // (undocumented) + get(taskId: string): Promise; + // (undocumented) + observe( + options: { + taskId: string; + after: number | undefined; + }, + callback: ( + error: Error | undefined, + result: { + events: DbTaskEventRow[]; + }, + ) => void, + ): () => void; + // (undocumented) + vacuumTasks(timeoutS: { timeoutS: number }): Promise; +} + +// @public +export type TaskEventType = 'completion' | 'log'; + +// @public +export type TaskSecrets = { + token: string | undefined; +}; + +// @public +export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3; + +// @public +export interface TaskSpecV1beta2 { + // (undocumented) + apiVersion: 'backstage.io/v1beta2'; + // (undocumented) + baseUrl?: string; + // (undocumented) + output: { + [name: string]: string; + }; + // (undocumented) + steps: Array<{ + id: string; + name: string; + action: string; + input?: JsonObject; + if?: string | boolean; + }>; + // (undocumented) + values: JsonObject; +} + +// @public +export interface TaskSpecV1beta3 { + // (undocumented) + apiVersion: 'scaffolder.backstage.io/v1beta3'; + // (undocumented) + baseUrl?: string; + // (undocumented) + output: { + [name: string]: JsonValue; + }; + // (undocumented) + parameters: JsonObject; + // Warning: (ae-forgotten-export) The symbol "TaskStep" needs to be exported by the entry point index.d.ts + // + // (undocumented) + steps: TaskStep[]; +} + +// @public +export interface TaskStore { + // (undocumented) + claimTask(): Promise; + // (undocumented) + completeTask(options: { + taskId: string; + status: Status; + eventBody: JsonObject; + }): Promise; + // (undocumented) + createTask( + task: TaskSpec, + secrets?: TaskSecrets, + ): Promise<{ + taskId: string; + }>; + // (undocumented) + emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise; + // (undocumented) + getTask(taskId: string): Promise; + // (undocumented) + heartbeatTask(taskId: string): Promise; + // (undocumented) + listEvents({ taskId, after }: TaskStoreGetEventsOptions): Promise<{ + events: DbTaskEventRow[]; + }>; + // (undocumented) + listStaleTasks(options: { timeoutS: number }): Promise<{ + tasks: { + taskId: string; + }[]; + }>; +} + +// @public +export type TaskStoreEmitOptions = { + taskId: string; + body: JsonObject; +}; + +// @public +export class TaskWorker { + // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts + constructor(options: Options); + // (undocumented) + runOneTask(task: Task): Promise; + // (undocumented) + start(): void; +} + // Warning: (ae-missing-release-tag) "TemplateAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -286,4 +591,8 @@ export class TemplateActionRegistry { action: TemplateAction, ): void; } + +// Warnings were encountered during analysis: +// +// src/scaffolder/tasks/DatabaseTaskStore.d.ts:51:9 - (ae-forgotten-export) The symbol "DbTaskEventRow" needs to be exported by the entry point index.d.ts ``` diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index 284e372b4c..cf2a554189 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -14,3 +14,27 @@ * limitations under the License. */ export * from './actions'; +export { + TaskAgent, + DatabaseTaskStore, + StorageTaskBroker, + TaskWorker, + LegacyWorkflowRunner, + DefaultWorkflowRunner, +} from './tasks'; +export type { + Task, + TaskBroker, + TaskSpec, + TaskSecrets, + DispatchResult, + TaskStore, + TaskStoreEmitOptions, + DbTaskRow, + TaskSpecV1beta2, + TaskSpecV1beta3, + Status, + TaskEventType, +} from './tasks/types'; + +export type { RawDbTaskEventRow } from './tasks/DatabaseTaskStore'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 09cba596b1..8f659cad02 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -46,6 +46,10 @@ export type RawDbTaskRow = { secrets?: string; }; +/** + * RawDbTaskEventRow + * @public + */ export type RawDbTaskEventRow = { id: number; task_id: string; @@ -54,6 +58,10 @@ export type RawDbTaskEventRow = { created_at: string; }; +/** + * DatabaseTaskStore + * @public + */ export class DatabaseTaskStore implements TaskStore { static async create(knex: Knex): Promise { await knex.migrate.latest({ diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index 2ac309a1d4..7b2b437be2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -77,6 +77,10 @@ const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { return { taskLogger, streamLogger }; }; +/* + * DefaultWorkflowRunner + * @public + */ export class DefaultWorkflowRunner implements WorkflowRunner { private readonly nunjucks: nunjucks.Environment; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts index 97e3a4dec4..e71fe596cd 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts @@ -43,9 +43,13 @@ type Options = { const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta2 => taskSpec.apiVersion === 'backstage.io/v1beta2'; + /** * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced * with the default workflow runner interface in the future so this entire thing can go bye bye. + * + * LegacyWorkflowRunner + * @public */ export class LegacyWorkflowRunner implements WorkflowRunner { private readonly handlebars: typeof Handlebars; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 867e5bb127..59f605702d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -27,6 +27,9 @@ import { DbTaskRow, } from './types'; +/* + * @public + */ export class TaskAgent implements Task { private isDone = false; @@ -117,10 +120,14 @@ function defer() { return { promise, resolve }; } +/** + * StorageTaskBroker + * @public + */ export class StorageTaskBroker implements TaskBroker { constructor( - private readonly storage: TaskStore, - private readonly logger: Logger, + protected readonly storage: TaskStore, + protected readonly logger: Logger, ) {} private deferredDispatch = defer(); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 7122c2fc3e..1f6d9d3381 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -24,6 +24,10 @@ type Options = { }; }; +/** + * TaskWorker + * @public + */ export class TaskWorker { constructor(private readonly options: Options) {} start() { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index dd13264aed..c068e8d7e1 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -15,5 +15,7 @@ */ export { DatabaseTaskStore } from './DatabaseTaskStore'; -export { StorageTaskBroker } from './StorageTaskBroker'; +export { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; export { TaskWorker } from './TaskWorker'; +export { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; +export { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index cd93404165..63b409e7ca 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -16,6 +16,10 @@ import { JsonValue, JsonObject } from '@backstage/config'; +/** + * Status + * @public + */ export type Status = | 'open' | 'processing' @@ -25,6 +29,10 @@ export type Status = export type CompletedTaskState = 'failed' | 'completed'; +/** + * DbTaskRow + * @public + */ export type DbTaskRow = { id: string; spec: TaskSpec; @@ -34,7 +42,16 @@ export type DbTaskRow = { secrets?: TaskSecrets; }; +/** + * TaskEventType + * @public + */ export type TaskEventType = 'completion' | 'log'; + +/** + * DbTaskEventRow + * @public + */ export type DbTaskEventRow = { id: number; taskId: string; @@ -43,6 +60,10 @@ export type DbTaskEventRow = { createdAt: string; }; +/** + * TaskSpecV1beta2 + * @public + */ export interface TaskSpecV1beta2 { apiVersion: 'backstage.io/v1beta2'; baseUrl?: string; @@ -64,6 +85,11 @@ export interface TaskStep { input?: JsonObject; if?: string | boolean; } + +/** + * TaskSpecV1beta3 + * @public + */ export interface TaskSpecV1beta3 { apiVersion: 'scaffolder.backstage.io/v1beta3'; baseUrl?: string; @@ -72,16 +98,32 @@ export interface TaskSpecV1beta3 { output: { [name: string]: JsonValue }; } +/** + * TaskSpec + * @public + */ export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3; +/** + * TaskSecrets + * @public + */ export type TaskSecrets = { token: string | undefined; }; +/** + * DispatchResult + * @public + */ export type DispatchResult = { taskId: string; }; +/** + * Task + * @public + */ export interface Task { spec: TaskSpec; secrets?: TaskSecrets; @@ -91,9 +133,16 @@ export interface Task { getWorkspaceName(): Promise; } +/** + * TaskBroker + * @public + */ export interface TaskBroker { claim(): Promise; - dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; + dispatch( + spec: TaskSpec, + secretTaskSecretss?: TaskSecrets, + ): Promise; vacuumTasks(timeoutS: { timeoutS: number }): Promise; observe( options: { @@ -105,8 +154,14 @@ export interface TaskBroker { result: { events: DbTaskEventRow[] }, ) => void, ): () => void; + + get(taskId: string): Promise; } +/** + * TaskStoreEmitOptions + * @public + */ export type TaskStoreEmitOptions = { taskId: string; body: JsonObject; @@ -117,6 +172,10 @@ export type TaskStoreGetEventsOptions = { after?: number | undefined; }; +/** + * TaskStore + * @public + */ export interface TaskStore { createTask( task: TaskSpec, @@ -142,6 +201,10 @@ export interface TaskStore { } export type WorkflowResponse = { output: { [key: string]: JsonValue } }; +/* + * WorkflowRunner + * @public + */ export interface WorkflowRunner { execute(task: Task): Promise; } diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index f8ba99336f..5f855a6e88 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -25,7 +25,13 @@ import { StorageTaskBroker, TaskWorker, } from '../scaffolder/tasks'; -import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry'; +import { + TemplateActionRegistry, + TemplateAction, + createBuiltinActions, + TaskBroker, + TaskSpec, +} from '../scaffolder'; import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { ContainerRunner, @@ -38,11 +44,9 @@ import { TemplateEntityV1beta2, Entity } from '@backstage/catalog-model'; import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; import { ScmIntegrations } from '@backstage/integration'; -import { TemplateAction } from '../scaffolder/actions'; -import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; + import { LegacyWorkflowRunner } from '../scaffolder/tasks/LegacyWorkflowRunner'; import { DefaultWorkflowRunner } from '../scaffolder/tasks/DefaultWorkflowRunner'; -import { TaskSpec } from '../scaffolder/tasks/types'; export interface RouterOptions { logger: Logger; @@ -53,6 +57,7 @@ export interface RouterOptions { actions?: TemplateAction[]; taskWorkers?: number; containerRunner: ContainerRunner; + taskBroker?: TaskBroker; } function isSupportedTemplate( @@ -89,7 +94,8 @@ export async function createRouter( const databaseTaskStore = await DatabaseTaskStore.create( await database.getClient(), ); - const taskBroker = new StorageTaskBroker(databaseTaskStore, logger); + const taskBroker = + options.taskBroker || new StorageTaskBroker(databaseTaskStore, logger); const actionRegistry = new TemplateActionRegistry(); const legacyWorkflowRunner = new LegacyWorkflowRunner({ logger, From 01a4c5f9987937746b45dc2c98c35890d343e8ca Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Fri, 15 Oct 2021 10:36:49 +0100 Subject: [PATCH 02/58] fixes tests Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/src/service/router.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 5f855a6e88..afc9cee986 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -25,13 +25,8 @@ import { StorageTaskBroker, TaskWorker, } from '../scaffolder/tasks'; -import { - TemplateActionRegistry, - TemplateAction, - createBuiltinActions, - TaskBroker, - TaskSpec, -} from '../scaffolder'; +import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry'; +import { TaskBroker } from '../scaffolder'; import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { ContainerRunner, @@ -44,9 +39,12 @@ import { TemplateEntityV1beta2, Entity } from '@backstage/catalog-model'; import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; import { ScmIntegrations } from '@backstage/integration'; +import { TemplateAction } from '../scaffolder/actions'; +import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; import { LegacyWorkflowRunner } from '../scaffolder/tasks/LegacyWorkflowRunner'; import { DefaultWorkflowRunner } from '../scaffolder/tasks/DefaultWorkflowRunner'; +import { TaskSpec } from '../scaffolder/tasks/types'; export interface RouterOptions { logger: Logger; From 5a58092168be995a31d9d49bb26b7e70caec068c Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 21 Oct 2021 16:32:45 +0100 Subject: [PATCH 03/58] no longer expose the storage task broker We can implement the task broker completely, so it is not neccessary to export it. This commit also addresses some review comments. Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/api-report.md | 178 ++++++++---------- plugins/scaffolder-backend/src/index.ts | 7 + .../src/scaffolder/index.ts | 25 +-- .../src/scaffolder/tasks/DatabaseTaskStore.ts | 17 +- .../scaffolder/tasks/DefaultWorkflowRunner.ts | 8 +- .../scaffolder/tasks/LegacyWorkflowRunner.ts | 4 - .../tasks/StorageTaskBroker.test.ts | 6 +- .../src/scaffolder/tasks/StorageTaskBroker.ts | 27 +-- .../src/scaffolder/tasks/TaskWorker.ts | 58 +++++- .../src/scaffolder/tasks/index.ts | 4 +- .../src/scaffolder/tasks/types.ts | 56 +++--- .../scaffolder-backend/src/service/router.ts | 9 +- 12 files changed, 215 insertions(+), 184 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index e0e7ad7ee5..f47c550875 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -26,7 +26,6 @@ import { ScmIntegrationRegistry } from '@backstage/integration'; import { ScmIntegrations } from '@backstage/integration'; import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; import { UrlReader } from '@backstage/backend-common'; -import * as winston from 'winston'; import { Writable } from 'stream'; // Warning: (ae-forgotten-export) The symbol "InputBase" needs to be exported by the entry point index.d.ts @@ -57,6 +56,9 @@ export class CatalogEntityClient { ): Promise; } +// @public +export type CompletedTaskState = 'failed' | 'completed'; + // Warning: (ae-missing-release-tag) "createBuiltinActions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -191,11 +193,22 @@ export const createTemplateAction: < templateAction: TemplateAction, ) => TemplateAction; +// Warning: (ae-missing-release-tag) "CreateWorkerOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type CreateWorkerOptions = { + taskBroker: TaskBroker; + actionRegistry: TemplateActionRegistry; + integrations: ScmIntegrations; + workingDirectory: string; + logger: Logger_2; +}; + // @public export class DatabaseTaskStore implements TaskStore { constructor(db: Knex); // (undocumented) - claimTask(): Promise; + claimTask(): Promise; // (undocumented) completeTask({ taskId, @@ -218,14 +231,12 @@ export class DatabaseTaskStore implements TaskStore { // (undocumented) emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise; // (undocumented) - getTask(taskId: string): Promise; + getTask(taskId: string): Promise; // (undocumented) heartbeatTask(taskId: string): Promise; - // Warning: (ae-forgotten-export) The symbol "TaskStoreGetEventsOptions" needs to be exported by the entry point index.d.ts - // // (undocumented) - listEvents({ taskId, after }: TaskStoreGetEventsOptions): Promise<{ - events: DbTaskEventRow[]; + listEvents({ taskId, after }: TaskStoreListEventsOptions): Promise<{ + events: SerializedTaskEvent[]; }>; // (undocumented) listStaleTasks({ timeoutS }: { timeoutS: number }): Promise<{ @@ -235,29 +246,6 @@ export class DatabaseTaskStore implements TaskStore { }>; } -// @public -export type DbTaskRow = { - id: string; - spec: TaskSpec; - status: Status; - createdAt: string; - lastHeartbeatAt?: string; - secrets?: TaskSecrets; -}; - -// Warning: (ae-forgotten-export) The symbol "WorkflowRunner" needs to be exported by the entry point index.d.ts -// Warning: (ae-missing-release-tag) "DefaultWorkflowRunner" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export class DefaultWorkflowRunner implements WorkflowRunner { - // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts - constructor(options: Options_3); - // Warning: (ae-forgotten-export) The symbol "WorkflowResponse" needs to be exported by the entry point index.d.ts - // - // (undocumented) - execute(task: Task): Promise; -} - // @public export type DispatchResult = { taskId: string; @@ -280,14 +268,6 @@ export function fetchContents({ outputPath: string; }): Promise; -// @public -export class LegacyWorkflowRunner implements WorkflowRunner { - // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts - constructor(options: Options_2); - // (undocumented) - execute(task: Task): Promise; -} - // Warning: (ae-missing-release-tag) "OctokitProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -299,17 +279,6 @@ export class OctokitProvider { } // @public -export type RawDbTaskEventRow = { - id: number; - task_id: string; - body: string; - event_type: TaskEventType; - created_at: string; -}; - -// Warning: (ae-missing-release-tag) "RouterOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) export interface RouterOptions { // (undocumented) actions?: TemplateAction[]; @@ -353,6 +322,25 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { validateEntityKind(entity: Entity): Promise; } +// @public +export type SerializedTask = { + id: string; + spec: TaskSpec; + status: Status; + createdAt: string; + lastHeartbeatAt?: string; + secrets?: TaskSecrets; +}; + +// @public +export type SerializedTaskEvent = { + id: number; + taskId: string; + body: JsonObject; + type: TaskEventType; + createdAt: string; +}; + // @public export type Status = | 'open' @@ -361,40 +349,8 @@ export type Status = | 'cancelled' | 'completed'; -// @public -export class StorageTaskBroker implements TaskBroker { - constructor(storage: TaskStore, logger: Logger_2); - // (undocumented) - claim(): Promise; - // (undocumented) - dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; - // (undocumented) - get(taskId: string): Promise; - // (undocumented) - protected readonly logger: Logger_2; - // (undocumented) - observe( - options: { - taskId: string; - after: number | undefined; - }, - callback: ( - error: Error | undefined, - result: { - events: DbTaskEventRow[]; - }, - ) => void, - ): () => void; - // (undocumented) - protected readonly storage: TaskStore; - // (undocumented) - vacuumTasks(timeoutS: { timeoutS: number }): Promise; -} - // @public export interface Task { - // Warning: (ae-forgotten-export) The symbol "CompletedTaskState" needs to be exported by the entry point index.d.ts - // // (undocumented) complete(result: CompletedTaskState, metadata?: JsonValue): Promise; // (undocumented) @@ -409,14 +365,10 @@ export interface Task { spec: TaskSpec; } -// Warning: (ae-missing-release-tag) "TaskAgent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export class TaskAgent implements Task { // (undocumented) complete(result: CompletedTaskState, metadata?: JsonObject): Promise; - // Warning: (ae-forgotten-export) The symbol "TaskState" needs to be exported by the entry point index.d.ts - // // (undocumented) static create( state: TaskState, @@ -440,12 +392,9 @@ export interface TaskBroker { // (undocumented) claim(): Promise; // (undocumented) - dispatch( - spec: TaskSpec, - secretTaskSecretss?: TaskSecrets, - ): Promise; + dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; // (undocumented) - get(taskId: string): Promise; + get(taskId: string): Promise; // (undocumented) observe( options: { @@ -455,7 +404,7 @@ export interface TaskBroker { callback: ( error: Error | undefined, result: { - events: DbTaskEventRow[]; + events: SerializedTaskEvent[]; }, ) => void, ): () => void; @@ -463,9 +412,6 @@ export interface TaskBroker { vacuumTasks(timeoutS: { timeoutS: number }): Promise; } -// @public -export type TaskEventType = 'completion' | 'log'; - // @public export type TaskSecrets = { token: string | undefined; @@ -514,10 +460,20 @@ export interface TaskSpecV1beta3 { steps: TaskStep[]; } +// @public +export interface TaskState { + // (undocumented) + secrets?: TaskSecrets; + // (undocumented) + spec: TaskSpec; + // (undocumented) + taskId: string; +} + // @public export interface TaskStore { // (undocumented) - claimTask(): Promise; + claimTask(): Promise; // (undocumented) completeTask(options: { taskId: string; @@ -534,12 +490,12 @@ export interface TaskStore { // (undocumented) emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise; // (undocumented) - getTask(taskId: string): Promise; + getTask(taskId: string): Promise; // (undocumented) heartbeatTask(taskId: string): Promise; // (undocumented) - listEvents({ taskId, after }: TaskStoreGetEventsOptions): Promise<{ - events: DbTaskEventRow[]; + listEvents({ taskId, after }: TaskStoreListEventsOptions): Promise<{ + events: SerializedTaskEvent[]; }>; // (undocumented) listStaleTasks(options: { timeoutS: number }): Promise<{ @@ -555,16 +511,32 @@ export type TaskStoreEmitOptions = { body: JsonObject; }; +// @public +export type TaskStoreListEventsOptions = { + taskId: string; + after?: number | undefined; +}; + // @public export class TaskWorker { - // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts - constructor(options: Options); + constructor(options: TaskWorkerOptions); + // (undocumented) + static createWorker(options: CreateWorkerOptions): TaskWorker; // (undocumented) runOneTask(task: Task): Promise; // (undocumented) start(): void; } +// @public +export type TaskWorkerOptions = { + taskBroker: TaskBroker; + runners: { + legacyWorkflowRunner: LegacyWorkflowRunner; + workflowRunner: WorkflowRunner; + }; +}; + // Warning: (ae-missing-release-tag) "TemplateAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -594,5 +566,7 @@ export class TemplateActionRegistry { // Warnings were encountered during analysis: // -// src/scaffolder/tasks/DatabaseTaskStore.d.ts:51:9 - (ae-forgotten-export) The symbol "DbTaskEventRow" needs to be exported by the entry point index.d.ts +// src/scaffolder/tasks/TaskWorker.d.ts:14:9 - (ae-forgotten-export) The symbol "LegacyWorkflowRunner" needs to be exported by the entry point index.d.ts +// src/scaffolder/tasks/TaskWorker.d.ts:15:9 - (ae-forgotten-export) The symbol "WorkflowRunner" needs to be exported by the entry point index.d.ts +// src/scaffolder/tasks/types.d.ts:37:5 - (ae-forgotten-export) The symbol "TaskEventType" needs to be exported by the entry point index.d.ts ``` diff --git a/plugins/scaffolder-backend/src/index.ts b/plugins/scaffolder-backend/src/index.ts index 76cda63eef..4c31e9f982 100644 --- a/plugins/scaffolder-backend/src/index.ts +++ b/plugins/scaffolder-backend/src/index.ts @@ -24,3 +24,10 @@ export * from './scaffolder'; export * from './service/router'; export * from './lib/catalog'; export * from './processor'; +export { TaskAgent } from './scaffolder/tasks'; +export type { + TaskBroker, + DispatchResult, + TaskStore, + Task, +} from './scaffolder/tasks/types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index cf2a554189..b2599f20ed 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -14,27 +14,18 @@ * limitations under the License. */ export * from './actions'; -export { - TaskAgent, - DatabaseTaskStore, - StorageTaskBroker, - TaskWorker, - LegacyWorkflowRunner, - DefaultWorkflowRunner, -} from './tasks'; +export { DatabaseTaskStore, TaskWorker } from './tasks'; +export type { TaskWorkerOptions, CreateWorkerOptions } from './tasks'; +export type { TaskState } from './tasks'; export type { - Task, - TaskBroker, - TaskSpec, TaskSecrets, - DispatchResult, - TaskStore, + TaskSpec, + CompletedTaskState, TaskStoreEmitOptions, - DbTaskRow, + TaskStoreListEventsOptions, + SerializedTask, + SerializedTaskEvent, TaskSpecV1beta2, TaskSpecV1beta3, Status, - TaskEventType, } from './tasks/types'; - -export type { RawDbTaskEventRow } from './tasks/DatabaseTaskStore'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 8f659cad02..3aec10d949 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -20,15 +20,15 @@ import { ConflictError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; import { - DbTaskEventRow, - DbTaskRow, + SerializedTaskEvent, + SerializedTask, Status, TaskEventType, TaskSecrets, TaskSpec, TaskStore, TaskStoreEmitOptions, - TaskStoreGetEventsOptions, + TaskStoreListEventsOptions, } from './types'; import { DateTime } from 'luxon'; @@ -46,10 +46,6 @@ export type RawDbTaskRow = { secrets?: string; }; -/** - * RawDbTaskEventRow - * @public - */ export type RawDbTaskEventRow = { id: number; task_id: string; @@ -60,6 +56,7 @@ export type RawDbTaskEventRow = { /** * DatabaseTaskStore + * * @public */ export class DatabaseTaskStore implements TaskStore { @@ -72,7 +69,7 @@ export class DatabaseTaskStore implements TaskStore { constructor(private readonly db: Knex) {} - async getTask(taskId: string): Promise { + async getTask(taskId: string): Promise { const [result] = await this.db('tasks') .where({ id: taskId }) .select(); @@ -109,7 +106,7 @@ export class DatabaseTaskStore implements TaskStore { return { taskId }; } - async claimTask(): Promise { + async claimTask(): Promise { return this.db.transaction(async tx => { const [task] = await tx('tasks') .where({ @@ -251,7 +248,7 @@ export class DatabaseTaskStore implements TaskStore { async listEvents({ taskId, after, - }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }> { + }: TaskStoreListEventsOptions): Promise<{ events: SerializedTaskEvent[] }> { const rawEvents = await this.db('task_events') .where({ task_id: taskId, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts index 7b2b437be2..145ded6bc0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -34,7 +34,7 @@ import { validate as validateJsonSchema } from 'jsonschema'; import { parseRepoUrl } from '../actions/builtin/publish/util'; import { TemplateActionRegistry } from '../actions'; -type Options = { +type NunjucksWorkflowRunnerOptions = { workingDirectory: string; actionRegistry: TemplateActionRegistry; integrations: ScmIntegrations; @@ -77,10 +77,6 @@ const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { return { taskLogger, streamLogger }; }; -/* - * DefaultWorkflowRunner - * @public - */ export class DefaultWorkflowRunner implements WorkflowRunner { private readonly nunjucks: nunjucks.Environment; @@ -92,7 +88,7 @@ export class DefaultWorkflowRunner implements WorkflowRunner { }, }; - constructor(private readonly options: Options) { + constructor(private readonly options: NunjucksWorkflowRunnerOptions) { this.nunjucks = nunjucks.configure(this.nunjucksOptions); // TODO(blam): let's work out how we can deprecate these. diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts index e71fe596cd..97e3a4dec4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts @@ -43,13 +43,9 @@ type Options = { const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta2 => taskSpec.apiVersion === 'backstage.io/v1beta2'; - /** * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced * with the default workflow runner interface in the future so this entire thing can go bye bye. - * - * LegacyWorkflowRunner - * @public */ export class LegacyWorkflowRunner implements WorkflowRunner { private readonly handlebars: typeof Handlebars; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts index 1a4d0d68c5..183253ea0e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -18,7 +18,7 @@ import { getVoidLogger, DatabaseManager } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { DatabaseTaskStore } from './DatabaseTaskStore'; import { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; -import { TaskSecrets, TaskSpec, DbTaskEventRow } from './types'; +import { TaskSecrets, TaskSpec, SerializedTaskEvent } from './types'; async function createStore(): Promise { const manager = DatabaseManager.fromConfig( @@ -127,8 +127,8 @@ describe('StorageTaskBroker', () => { const { taskId } = await broker1.dispatch({} as TaskSpec); - const logPromise = new Promise(resolve => { - const observedEvents = new Array(); + const logPromise = new Promise(resolve => { + const observedEvents = new Array(); broker2.observe({ taskId, after: undefined }, (_err, { events }) => { observedEvents.push(...events); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 59f605702d..af1f3752e1 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -23,11 +23,13 @@ import { TaskStore, TaskBroker, DispatchResult, - DbTaskEventRow, - DbTaskRow, + SerializedTaskEvent, + SerializedTask, } from './types'; -/* +/** + * TaskAgent + * * @public */ export class TaskAgent implements Task { @@ -106,7 +108,12 @@ export class TaskAgent implements Task { } } -interface TaskState { +/** + * TaskState + * + * @public + */ +export interface TaskState { spec: TaskSpec; taskId: string; secrets?: TaskSecrets; @@ -120,14 +127,10 @@ function defer() { return { promise, resolve }; } -/** - * StorageTaskBroker - * @public - */ export class StorageTaskBroker implements TaskBroker { constructor( - protected readonly storage: TaskStore, - protected readonly logger: Logger, + private readonly storage: TaskStore, + private readonly logger: Logger, ) {} private deferredDispatch = defer(); @@ -161,7 +164,7 @@ export class StorageTaskBroker implements TaskBroker { }; } - async get(taskId: string): Promise { + async get(taskId: string): Promise { return this.storage.getTask(taskId); } @@ -172,7 +175,7 @@ export class StorageTaskBroker implements TaskBroker { }, callback: ( error: Error | undefined, - result: { events: DbTaskEventRow[] }, + result: { events: SerializedTaskEvent[] }, ) => void, ): () => void { const { taskId } = options; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 1f6d9d3381..65ba71e9fe 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -13,10 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Task, TaskBroker, WorkflowRunner } from './types'; import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; +import { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; +import { Logger } from 'winston'; +import { TemplateActionRegistry } from '../actions'; +import { ScmIntegrations } from '@backstage/integration'; -type Options = { +/** + * TaskWorkerOptions + * + * @public + */ +export type TaskWorkerOptions = { taskBroker: TaskBroker; runners: { legacyWorkflowRunner: LegacyWorkflowRunner; @@ -24,12 +34,56 @@ type Options = { }; }; +/** + * CreateWorkerOptions + * + * @public + */ +export type CreateWorkerOptions = { + taskBroker: TaskBroker; + actionRegistry: TemplateActionRegistry; + integrations: ScmIntegrations; + workingDirectory: string; + logger: Logger; +}; + /** * TaskWorker + * * @public */ export class TaskWorker { - constructor(private readonly options: Options) {} + constructor(private readonly options: TaskWorkerOptions) {} + + static createWorker(options: CreateWorkerOptions) { + const { + taskBroker, + logger, + actionRegistry, + integrations, + workingDirectory, + } = options; + + const legacyWorkflowRunner = new LegacyWorkflowRunner({ + logger, + actionRegistry, + integrations, + workingDirectory, + }); + + const workflowRunner = new DefaultWorkflowRunner({ + actionRegistry, + integrations, + logger, + workingDirectory, + }); + + return new TaskWorker({ + taskBroker: taskBroker, + runners: { legacyWorkflowRunner, workflowRunner }, + }); + } + start() { (async () => { for (;;) { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index c068e8d7e1..ada12df476 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -16,6 +16,6 @@ export { DatabaseTaskStore } from './DatabaseTaskStore'; export { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; +export type { TaskState } from './StorageTaskBroker'; export { TaskWorker } from './TaskWorker'; -export { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; -export { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; +export type { TaskWorkerOptions, CreateWorkerOptions } from './TaskWorker'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 63b409e7ca..e30d156736 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -18,6 +18,7 @@ import { JsonValue, JsonObject } from '@backstage/config'; /** * Status + * * @public */ export type Status = @@ -27,13 +28,19 @@ export type Status = | 'cancelled' | 'completed'; +/** + * CompletedTaskState + * + * @public + */ export type CompletedTaskState = 'failed' | 'completed'; /** - * DbTaskRow + * SerializedTask + * * @public */ -export type DbTaskRow = { +export type SerializedTask = { id: string; spec: TaskSpec; status: Status; @@ -42,17 +49,14 @@ export type DbTaskRow = { secrets?: TaskSecrets; }; -/** - * TaskEventType - * @public - */ export type TaskEventType = 'completion' | 'log'; /** - * DbTaskEventRow + * SerializedTaskEvent + * * @public */ -export type DbTaskEventRow = { +export type SerializedTaskEvent = { id: number; taskId: string; body: JsonObject; @@ -62,6 +66,7 @@ export type DbTaskEventRow = { /** * TaskSpecV1beta2 + * * @public */ export interface TaskSpecV1beta2 { @@ -88,6 +93,7 @@ export interface TaskStep { /** * TaskSpecV1beta3 + * * @public */ export interface TaskSpecV1beta3 { @@ -100,12 +106,14 @@ export interface TaskSpecV1beta3 { /** * TaskSpec + * * @public */ export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3; /** * TaskSecrets + * * @public */ export type TaskSecrets = { @@ -114,6 +122,7 @@ export type TaskSecrets = { /** * DispatchResult + * * @public */ export type DispatchResult = { @@ -122,6 +131,7 @@ export type DispatchResult = { /** * Task + * * @public */ export interface Task { @@ -135,14 +145,12 @@ export interface Task { /** * TaskBroker + * * @public */ export interface TaskBroker { claim(): Promise; - dispatch( - spec: TaskSpec, - secretTaskSecretss?: TaskSecrets, - ): Promise; + dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; vacuumTasks(timeoutS: { timeoutS: number }): Promise; observe( options: { @@ -151,15 +159,15 @@ export interface TaskBroker { }, callback: ( error: Error | undefined, - result: { events: DbTaskEventRow[] }, + result: { events: SerializedTaskEvent[] }, ) => void, ): () => void; - - get(taskId: string): Promise; + get(taskId: string): Promise; } /** * TaskStoreEmitOptions + * * @public */ export type TaskStoreEmitOptions = { @@ -167,13 +175,19 @@ export type TaskStoreEmitOptions = { body: JsonObject; }; -export type TaskStoreGetEventsOptions = { +/** + * TaskStoreListEventsOptions + * + * @public + */ +export type TaskStoreListEventsOptions = { taskId: string; after?: number | undefined; }; /** * TaskStore + * * @public */ export interface TaskStore { @@ -181,8 +195,8 @@ export interface TaskStore { task: TaskSpec, secrets?: TaskSecrets, ): Promise<{ taskId: string }>; - getTask(taskId: string): Promise; - claimTask(): Promise; + getTask(taskId: string): Promise; + claimTask(): Promise; completeTask(options: { taskId: string; status: Status; @@ -197,14 +211,10 @@ export interface TaskStore { listEvents({ taskId, after, - }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }>; + }: TaskStoreListEventsOptions): Promise<{ events: SerializedTaskEvent[] }>; } export type WorkflowResponse = { output: { [key: string]: JsonValue } }; -/* - * WorkflowRunner - * @public - */ export interface WorkflowRunner { execute(task: Task): Promise; } diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index afc9cee986..d4d4dd0486 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -26,7 +26,6 @@ import { TaskWorker, } from '../scaffolder/tasks'; import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry'; -import { TaskBroker } from '../scaffolder'; import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { ContainerRunner, @@ -41,11 +40,15 @@ import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; import { ScmIntegrations } from '@backstage/integration'; import { TemplateAction } from '../scaffolder/actions'; import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; - import { LegacyWorkflowRunner } from '../scaffolder/tasks/LegacyWorkflowRunner'; import { DefaultWorkflowRunner } from '../scaffolder/tasks/DefaultWorkflowRunner'; -import { TaskSpec } from '../scaffolder/tasks/types'; +import { TaskBroker, TaskSpec } from '../scaffolder/tasks/types'; +/** + * RouterOptions + * + * @public + */ export interface RouterOptions { logger: Logger; config: Config; From 33d99f7cdd963da3421b4570446641dd0dc890dd Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Fri, 22 Oct 2021 08:12:29 +0100 Subject: [PATCH 04/58] data store options type added Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/api-report.md | 10 ++++---- .../src/scaffolder/tasks/DatabaseTaskStore.ts | 23 +++++++++++++++---- .../tasks/StorageTaskBroker.test.ts | 4 +++- .../src/scaffolder/tasks/TaskWorker.test.ts | 4 +++- .../src/scaffolder/tasks/index.ts | 2 +- .../scaffolder-backend/src/service/router.ts | 6 ++--- 6 files changed, 34 insertions(+), 15 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index f47c550875..deb2657c59 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -193,9 +193,7 @@ export const createTemplateAction: < templateAction: TemplateAction, ) => TemplateAction; -// Warning: (ae-missing-release-tag) "CreateWorkerOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export type CreateWorkerOptions = { taskBroker: TaskBroker; actionRegistry: TemplateActionRegistry; @@ -206,7 +204,7 @@ export type CreateWorkerOptions = { // @public export class DatabaseTaskStore implements TaskStore { - constructor(db: Knex); + constructor(options: DatabaseTaskStoreOptions); // (undocumented) claimTask(): Promise; // (undocumented) @@ -219,8 +217,10 @@ export class DatabaseTaskStore implements TaskStore { status: Status; eventBody: JsonObject; }): Promise; + // Warning: (ae-forgotten-export) The symbol "DatabaseTaskStoreOptions" needs to be exported by the entry point index.d.ts + // // (undocumented) - static create(knex: Knex): Promise; + static create(options: DatabaseTaskStoreOptions): Promise; // (undocumented) createTask( spec: TaskSpec, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 3aec10d949..6783da674e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -54,20 +54,35 @@ export type RawDbTaskEventRow = { created_at: string; }; +/** + * DatabaseTaskStore + * + * @public + */ +export type DatabaseTaskStoreOptions = { + database: Knex; +}; + /** * DatabaseTaskStore * * @public */ export class DatabaseTaskStore implements TaskStore { - static async create(knex: Knex): Promise { - await knex.migrate.latest({ + private readonly db: Knex; + + static async create( + options: DatabaseTaskStoreOptions, + ): Promise { + await options.database.migrate.latest({ directory: migrationsDir, }); - return new DatabaseTaskStore(knex); + return new DatabaseTaskStore(options); } - constructor(private readonly db: Knex) {} + constructor(options: DatabaseTaskStoreOptions) { + this.db = options.database; + } async getTask(taskId: string): Promise { const [result] = await this.db('tasks') diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts index 183253ea0e..c5d94c1324 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -31,7 +31,9 @@ async function createStore(): Promise { }, }), ).forPlugin('scaffolder'); - return await DatabaseTaskStore.create(await manager.getClient()); + return await DatabaseTaskStore.create({ + database: await manager.getClient(), + }); } describe('StorageTaskBroker', () => { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index 87a0229b5d..f6b332b6d3 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -33,7 +33,9 @@ async function createStore(): Promise { }, }), ).forPlugin('scaffolder'); - return await DatabaseTaskStore.create(await manager.getClient()); + return await DatabaseTaskStore.create({ + database: await manager.getClient(), + }); } describe('TaskWorker', () => { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index ada12df476..13831ca1bb 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - export { DatabaseTaskStore } from './DatabaseTaskStore'; +export type { DatabaseTaskStoreOptions } from './DatabaseTaskStore'; export { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; export type { TaskState } from './StorageTaskBroker'; export { TaskWorker } from './TaskWorker'; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index d4d4dd0486..0bc00a2778 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -92,9 +92,9 @@ export async function createRouter( const entityClient = new CatalogEntityClient(catalogClient); const integrations = ScmIntegrations.fromConfig(config); - const databaseTaskStore = await DatabaseTaskStore.create( - await database.getClient(), - ); + const databaseTaskStore = await DatabaseTaskStore.create({ + database: await database.getClient(), + }); const taskBroker = options.taskBroker || new StorageTaskBroker(databaseTaskStore, logger); const actionRegistry = new TemplateActionRegistry(); From 45a2a91a3165aca21aed793f7744326c9a55e29b Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Tue, 26 Oct 2021 09:48:56 +0100 Subject: [PATCH 05/58] update api reports after merge Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/api-report.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index b40270f317..a94fa8a2a0 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -14,11 +14,9 @@ import { createFetchCookiecutterAction } from '@backstage/plugin-scaffolder-back import { createPullRequest } from 'octokit-plugin-create-pull-request'; import { Entity } from '@backstage/catalog-model'; import express from 'express'; -import { JsonObject } from '@backstage/config'; -import { JsonObject as JsonObject_2 } from '@backstage/types'; -import { JsonValue } from '@backstage/config'; +import { JsonObject } from '@backstage/types'; +import { JsonValue } from '@backstage/types'; import { Knex } from 'knex'; -import { JsonValue as JsonValue_2 } from '@backstage/types'; import { LocationSpec } from '@backstage/catalog-model'; import { Logger as Logger_2 } from 'winston'; import { Octokit } from '@octokit/rest'; From d781df18437df3eb15149ce292158b0527f637ac Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Tue, 26 Oct 2021 11:34:13 +0100 Subject: [PATCH 06/58] hide workflow runners behind createWorker Also renames legacy and default workflow runners. Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/api-report.md | 19 +---- .../sample-templates/all-templates.yaml | 1 - .../bitbucket-demo/template/catalog-info.yaml | 4 +- .../src/scaffolder/index.ts | 3 +- .../tasks/DefaultWorkflowRunner.test.ts | 6 +- ...wRunner.ts => HandlebarsWorkflowRunner.ts} | 2 +- .../tasks/LegacyWorkflowRunner.test.ts | 6 +- ...lowRunner.ts => NunjucksWorkflowRunner.ts} | 2 +- .../src/scaffolder/tasks/TaskWorker.test.ts | 74 +++++++++++++------ .../src/scaffolder/tasks/TaskWorker.ts | 12 +-- .../src/scaffolder/tasks/types.ts | 5 ++ .../scaffolder-backend/src/service/router.ts | 25 ++----- 12 files changed, 82 insertions(+), 77 deletions(-) rename plugins/scaffolder-backend/src/scaffolder/tasks/{LegacyWorkflowRunner.ts => HandlebarsWorkflowRunner.ts} (99%) rename plugins/scaffolder-backend/src/scaffolder/tasks/{DefaultWorkflowRunner.ts => NunjucksWorkflowRunner.ts} (99%) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index a94fa8a2a0..d0f4cb36e8 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -412,6 +412,9 @@ export interface TaskBroker { vacuumTasks(timeoutS: { timeoutS: number }): Promise; } +// @public +export type TaskEventType = 'completion' | 'log'; + // @public export type TaskSecrets = { token: string | undefined; @@ -519,7 +522,6 @@ export type TaskStoreListEventsOptions = { // @public export class TaskWorker { - constructor(options: TaskWorkerOptions); // (undocumented) static createWorker(options: CreateWorkerOptions): TaskWorker; // (undocumented) @@ -528,15 +530,6 @@ export class TaskWorker { start(): void; } -// @public -export type TaskWorkerOptions = { - taskBroker: TaskBroker; - runners: { - legacyWorkflowRunner: LegacyWorkflowRunner; - workflowRunner: WorkflowRunner; - }; -}; - // Warning: (ae-missing-release-tag) "TemplateAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -563,10 +556,4 @@ export class TemplateActionRegistry { action: TemplateAction, ): void; } - -// Warnings were encountered during analysis: -// -// src/scaffolder/tasks/TaskWorker.d.ts:14:9 - (ae-forgotten-export) The symbol "LegacyWorkflowRunner" needs to be exported by the entry point index.d.ts -// src/scaffolder/tasks/TaskWorker.d.ts:15:9 - (ae-forgotten-export) The symbol "WorkflowRunner" needs to be exported by the entry point index.d.ts -// src/scaffolder/tasks/types.d.ts:37:5 - (ae-forgotten-export) The symbol "TaskEventType" needs to be exported by the entry point index.d.ts ``` diff --git a/plugins/scaffolder-backend/sample-templates/all-templates.yaml b/plugins/scaffolder-backend/sample-templates/all-templates.yaml index 2d5adb8e3b..6637c9479b 100644 --- a/plugins/scaffolder-backend/sample-templates/all-templates.yaml +++ b/plugins/scaffolder-backend/sample-templates/all-templates.yaml @@ -6,7 +6,6 @@ metadata: spec: targets: - ./remote-templates.yaml - # For local development of a template, you can reference your local templates here. # Examples: # diff --git a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml index 875664d2a8..3b685a57e0 100644 --- a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml +++ b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml @@ -1,8 +1,8 @@ apiVersion: backstage.io/v1alpha1 kind: Component metadata: - name: {{cookiecutter.name | jsonify}} + name: { { cookiecutter.name | jsonify } } spec: type: website lifecycle: experimental - owner: {{cookiecutter.owner | jsonify}} + owner: { { cookiecutter.owner | jsonify } } diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index b2599f20ed..6b0e033b7d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -15,7 +15,7 @@ */ export * from './actions'; export { DatabaseTaskStore, TaskWorker } from './tasks'; -export type { TaskWorkerOptions, CreateWorkerOptions } from './tasks'; +export type { CreateWorkerOptions } from './tasks'; export type { TaskState } from './tasks'; export type { TaskSecrets, @@ -28,4 +28,5 @@ export type { TaskSpecV1beta2, TaskSpecV1beta3, Status, + TaskEventType, } from './tasks/types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts index d6d1b0cfd2..0a843f4ddf 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -18,7 +18,7 @@ import mockFs from 'mock-fs'; import * as winston from 'winston'; import { getVoidLogger } from '@backstage/backend-common'; -import { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; +import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner'; import { TemplateActionRegistry } from '../actions'; import { ScmIntegrations } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; @@ -27,7 +27,7 @@ import { Task, TaskSpec } from './types'; describe('DefaultWorkflowRunner', () => { const logger = getVoidLogger(); let actionRegistry = new TemplateActionRegistry(); - let runner: DefaultWorkflowRunner; + let runner: NunjucksWorkflowRunner; let fakeActionHandler: jest.Mock; const integrations = ScmIntegrations.fromConfig( @@ -88,7 +88,7 @@ describe('DefaultWorkflowRunner', () => { }, }); - runner = new DefaultWorkflowRunner({ + runner = new NunjucksWorkflowRunner({ actionRegistry, integrations, workingDirectory: '/tmp', diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts similarity index 99% rename from plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts rename to plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts index 6d43ae978c..831ef98784 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts @@ -47,7 +47,7 @@ const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta2 => * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced * with the default workflow runner interface in the future so this entire thing can go bye bye. */ -export class LegacyWorkflowRunner implements WorkflowRunner { +export class HandlebarsWorkflowRunner implements WorkflowRunner { private readonly handlebars: typeof Handlebars; constructor(private readonly options: Options) { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts index 2714f1b80c..aa46747c68 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts @@ -20,12 +20,12 @@ import { createTemplateAction, TemplateActionRegistry } from '../actions'; import { ScmIntegrations } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; -import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; +import { HandlebarsWorkflowRunner } from './HandlebarsWorkflowRunner'; import { Task, TaskSpec } from './types'; import { RepoSpec } from '../actions/builtin/publish/util'; describe('LegacyWorkflowRunner', () => { - let runner: LegacyWorkflowRunner; + let runner: HandlebarsWorkflowRunner; const logger = getVoidLogger(); let actionRegistry = new TemplateActionRegistry(); @@ -60,7 +60,7 @@ describe('LegacyWorkflowRunner', () => { }, }); - runner = new LegacyWorkflowRunner({ + runner = new HandlebarsWorkflowRunner({ actionRegistry, integrations, workingDirectory: '/tmp', diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts similarity index 99% rename from plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts rename to plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index e56befd939..5cceb16324 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -77,7 +77,7 @@ const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { return { taskLogger, streamLogger }; }; -export class DefaultWorkflowRunner implements WorkflowRunner { +export class NunjucksWorkflowRunner implements WorkflowRunner { private readonly nunjucks: nunjucks.Environment; private readonly nunjucksOptions: nunjucks.ConfigureOptions = { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index f6b332b6d3..3cb73abb85 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -19,8 +19,20 @@ import { ConfigReader } from '@backstage/config'; import { DatabaseTaskStore } from './DatabaseTaskStore'; import { StorageTaskBroker } from './StorageTaskBroker'; import { TaskWorker } from './TaskWorker'; -import { WorkflowRunner } from './types'; -import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; +import { HandlebarsWorkflowRunner } from './HandlebarsWorkflowRunner'; +import { ScmIntegrations } from '@backstage/integration'; +import { TemplateActionRegistry } from '../actions'; +import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner'; + +jest.mock('./HandlebarsWorkflowRunner'); +const MockedHandlebarsWorkflowRunner = + HandlebarsWorkflowRunner as jest.Mock; +MockedHandlebarsWorkflowRunner.mockImplementation(); + +jest.mock('./NunjucksWorkflowRunner'); +const MockedNunjucksWorkflowRunner = + NunjucksWorkflowRunner as jest.Mock; +MockedNunjucksWorkflowRunner.mockImplementation(); async function createStore(): Promise { const manager = DatabaseManager.fromConfig( @@ -40,13 +52,19 @@ async function createStore(): Promise { describe('TaskWorker', () => { let storage: DatabaseTaskStore; - const workflowRunner: WorkflowRunner = { - execute: jest.fn(), - } as unknown as WorkflowRunner; - const legacyWorkflowRunner: LegacyWorkflowRunner = { + const integrations: ScmIntegrations = {} as ScmIntegrations; + + const actionRegistry: TemplateActionRegistry = {} as TemplateActionRegistry; + const workingDirectory = '/tmp/scaffolder'; + + const handlebarsWorkflowRunner: HandlebarsWorkflowRunner = { execute: jest.fn(), - } as unknown as LegacyWorkflowRunner; + } as unknown as HandlebarsWorkflowRunner; + + const workflowRunner: NunjucksWorkflowRunner = { + execute: jest.fn(), + } as unknown as NunjucksWorkflowRunner; beforeAll(async () => { storage = await createStore(); @@ -54,18 +72,22 @@ describe('TaskWorker', () => { beforeEach(() => { jest.resetAllMocks(); + MockedHandlebarsWorkflowRunner.mockImplementation( + () => handlebarsWorkflowRunner, + ); + MockedNunjucksWorkflowRunner.mockImplementation(() => workflowRunner); }); const logger = getVoidLogger(); it('should call the legacy workflow runner when the apiVersion is not beta3', async () => { const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ + const taskWorker = TaskWorker.createWorker({ + logger, + workingDirectory, + integrations, taskBroker: broker, - runners: { - legacyWorkflowRunner, - workflowRunner, - }, + actionRegistry, }); await broker.dispatch({ @@ -80,17 +102,23 @@ describe('TaskWorker', () => { const task = await broker.claim(); await taskWorker.runOneTask(task); - expect(legacyWorkflowRunner.execute).toHaveBeenCalled(); + expect(MockedHandlebarsWorkflowRunner).toBeCalledWith({ + actionRegistry, + integrations, + logger, + workingDirectory, + }); + expect(handlebarsWorkflowRunner.execute).toHaveBeenCalled(); }); it('should call the default workflow runner when the apiVersion is beta3', async () => { const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ + const taskWorker = TaskWorker.createWorker({ + logger, + workingDirectory, + integrations, taskBroker: broker, - runners: { - legacyWorkflowRunner, - workflowRunner, - }, + actionRegistry, }); await broker.dispatch({ @@ -114,12 +142,12 @@ describe('TaskWorker', () => { }); const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ + const taskWorker = TaskWorker.createWorker({ + logger, + workingDirectory, + integrations, taskBroker: broker, - runners: { - legacyWorkflowRunner, - workflowRunner, - }, + actionRegistry, }); const { taskId } = await broker.dispatch({ diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 6e70ea06a6..fcc215be99 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -15,8 +15,8 @@ */ import { Task, TaskBroker, WorkflowRunner } from './types'; -import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; -import { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; +import { HandlebarsWorkflowRunner } from './HandlebarsWorkflowRunner'; +import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner'; import { Logger } from 'winston'; import { TemplateActionRegistry } from '../actions'; import { ScmIntegrations } from '@backstage/integration'; @@ -30,7 +30,7 @@ import { assertError } from '@backstage/errors'; export type TaskWorkerOptions = { taskBroker: TaskBroker; runners: { - legacyWorkflowRunner: LegacyWorkflowRunner; + legacyWorkflowRunner: HandlebarsWorkflowRunner; workflowRunner: WorkflowRunner; }; }; @@ -54,7 +54,7 @@ export type CreateWorkerOptions = { * @public */ export class TaskWorker { - constructor(private readonly options: TaskWorkerOptions) {} + private constructor(private readonly options: TaskWorkerOptions) {} static createWorker(options: CreateWorkerOptions) { const { @@ -65,14 +65,14 @@ export class TaskWorker { workingDirectory, } = options; - const legacyWorkflowRunner = new LegacyWorkflowRunner({ + const legacyWorkflowRunner = new HandlebarsWorkflowRunner({ logger, actionRegistry, integrations, workingDirectory, }); - const workflowRunner = new DefaultWorkflowRunner({ + const workflowRunner = new NunjucksWorkflowRunner({ actionRegistry, integrations, logger, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index cd4d291719..f2e13923ec 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -49,6 +49,11 @@ export type SerializedTask = { secrets?: TaskSecrets; }; +/** + * TaskEventType + * + * @public + */ export type TaskEventType = 'completion' | 'log'; /** diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 0bc00a2778..fcd5551be2 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -40,8 +40,6 @@ import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; import { ScmIntegrations } from '@backstage/integration'; import { TemplateAction } from '../scaffolder/actions'; import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; -import { LegacyWorkflowRunner } from '../scaffolder/tasks/LegacyWorkflowRunner'; -import { DefaultWorkflowRunner } from '../scaffolder/tasks/DefaultWorkflowRunner'; import { TaskBroker, TaskSpec } from '../scaffolder/tasks/types'; /** @@ -98,28 +96,15 @@ export async function createRouter( const taskBroker = options.taskBroker || new StorageTaskBroker(databaseTaskStore, logger); const actionRegistry = new TemplateActionRegistry(); - const legacyWorkflowRunner = new LegacyWorkflowRunner({ - logger, - actionRegistry, - integrations, - workingDirectory, - }); - - const workflowRunner = new DefaultWorkflowRunner({ - actionRegistry, - integrations, - logger, - workingDirectory, - }); const workers = []; for (let i = 0; i < (taskWorkers || 1); i++) { - const worker = new TaskWorker({ + const worker = TaskWorker.createWorker({ taskBroker, - runners: { - legacyWorkflowRunner, - workflowRunner, - }, + actionRegistry, + integrations, + logger, + workingDirectory, }); workers.push(worker); } From c754c390ee16c4a89a6609fc057772ae9ffd9217 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Tue, 26 Oct 2021 12:09:54 +0100 Subject: [PATCH 07/58] fix up exports to follow covention Signed-off-by: Brian Fletcher --- .../scaffolder-backend/src/scaffolder/index.ts | 17 +---------------- .../src/scaffolder/tasks/index.ts | 18 +++++++++++++++--- .../scaffolder-backend/src/service/router.ts | 10 +++++----- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index 6b0e033b7d..e055223d8c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -14,19 +14,4 @@ * limitations under the License. */ export * from './actions'; -export { DatabaseTaskStore, TaskWorker } from './tasks'; -export type { CreateWorkerOptions } from './tasks'; -export type { TaskState } from './tasks'; -export type { - TaskSecrets, - TaskSpec, - CompletedTaskState, - TaskStoreEmitOptions, - TaskStoreListEventsOptions, - SerializedTask, - SerializedTaskEvent, - TaskSpecV1beta2, - TaskSpecV1beta3, - Status, - TaskEventType, -} from './tasks/types'; +export * from './tasks'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index 13831ca1bb..d99cbcda6e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -14,8 +14,20 @@ * limitations under the License. */ export { DatabaseTaskStore } from './DatabaseTaskStore'; -export type { DatabaseTaskStoreOptions } from './DatabaseTaskStore'; -export { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; +export { TaskAgent } from './StorageTaskBroker'; export type { TaskState } from './StorageTaskBroker'; export { TaskWorker } from './TaskWorker'; -export type { TaskWorkerOptions, CreateWorkerOptions } from './TaskWorker'; +export type { CreateWorkerOptions } from './TaskWorker'; +export type { + TaskSecrets, + TaskSpec, + CompletedTaskState, + TaskStoreEmitOptions, + TaskStoreListEventsOptions, + SerializedTask, + SerializedTaskEvent, + TaskSpecV1beta2, + TaskSpecV1beta3, + Status, + TaskEventType, +} from './types'; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index fcd5551be2..700f2c6e6f 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -22,10 +22,12 @@ import { CatalogEntityClient } from '../lib/catalog'; import { validate } from 'jsonschema'; import { DatabaseTaskStore, - StorageTaskBroker, + TemplateActionRegistry, TaskWorker, -} from '../scaffolder/tasks'; -import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry'; + TemplateAction, + createBuiltinActions, +} from '../scaffolder'; +import { StorageTaskBroker } from '../scaffolder/tasks/StorageTaskBroker'; import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { ContainerRunner, @@ -38,8 +40,6 @@ import { TemplateEntityV1beta2, Entity } from '@backstage/catalog-model'; import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; import { ScmIntegrations } from '@backstage/integration'; -import { TemplateAction } from '../scaffolder/actions'; -import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; import { TaskBroker, TaskSpec } from '../scaffolder/tasks/types'; /** From f49099fc13ea584f8d6807c4b6734ebc7d08db75 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Tue, 26 Oct 2021 15:42:02 +0100 Subject: [PATCH 08/58] fix more imports to follow convention Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/src/index.ts | 7 ------- plugins/scaffolder-backend/src/scaffolder/tasks/index.ts | 4 ++++ plugins/scaffolder-backend/src/service/router.test.ts | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/plugins/scaffolder-backend/src/index.ts b/plugins/scaffolder-backend/src/index.ts index 4c31e9f982..76cda63eef 100644 --- a/plugins/scaffolder-backend/src/index.ts +++ b/plugins/scaffolder-backend/src/index.ts @@ -24,10 +24,3 @@ export * from './scaffolder'; export * from './service/router'; export * from './lib/catalog'; export * from './processor'; -export { TaskAgent } from './scaffolder/tasks'; -export type { - TaskBroker, - DispatchResult, - TaskStore, - Task, -} from './scaffolder/tasks/types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index d99cbcda6e..941c27ef0e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -30,4 +30,8 @@ export type { TaskSpecV1beta3, Status, TaskEventType, + TaskBroker, + Task, + TaskStore, + DispatchResult, } from './types'; diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index c0fdcb2ef7..71b6d27b5e 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -40,7 +40,7 @@ import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; -import { createRouter } from './router'; +import { createRouter } from '../index'; const createCatalogClient = (templates: any[] = []) => ({ From 50eba2374b4d4ce2033714a94b0fe6b056adde9f Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 27 Oct 2021 09:16:23 +0100 Subject: [PATCH 09/58] revert prettier change to scaffolder examples Signed-off-by: Brian Fletcher --- .../bitbucket-demo/template/catalog-info.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml index 3b685a57e0..875664d2a8 100644 --- a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml +++ b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template/catalog-info.yaml @@ -1,8 +1,8 @@ apiVersion: backstage.io/v1alpha1 kind: Component metadata: - name: { { cookiecutter.name | jsonify } } + name: {{cookiecutter.name | jsonify}} spec: type: website lifecycle: experimental - owner: { { cookiecutter.owner | jsonify } } + owner: {{cookiecutter.owner | jsonify}} From 5a98c405f52b070918afb44f7116ecbaa6a6432a Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 27 Oct 2021 10:48:17 +0100 Subject: [PATCH 10/58] rename task agent to task manager Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/api-report.md | 80 +++++++++---------- .../tasks/DefaultWorkflowRunner.test.ts | 4 +- .../tasks/HandlebarsWorkflowRunner.ts | 4 +- .../tasks/LegacyWorkflowRunner.test.ts | 4 +- .../tasks/NunjucksWorkflowRunner.ts | 12 ++- .../tasks/StorageTaskBroker.test.ts | 12 +-- .../src/scaffolder/tasks/StorageTaskBroker.ts | 12 +-- .../src/scaffolder/tasks/TaskWorker.ts | 4 +- .../src/scaffolder/tasks/index.ts | 4 +- .../src/scaffolder/tasks/types.ts | 6 +- 10 files changed, 74 insertions(+), 68 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index d0f4cb36e8..5441c4de6b 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -349,48 +349,10 @@ export type Status = | 'cancelled' | 'completed'; -// @public -export interface Task { - // (undocumented) - complete(result: CompletedTaskState, metadata?: JsonValue): Promise; - // (undocumented) - done: boolean; - // (undocumented) - emitLog(message: string, metadata?: JsonValue): Promise; - // (undocumented) - getWorkspaceName(): Promise; - // (undocumented) - secrets?: TaskSecrets; - // (undocumented) - spec: TaskSpec; -} - -// @public -export class TaskAgent implements Task { - // (undocumented) - complete(result: CompletedTaskState, metadata?: JsonObject): Promise; - // (undocumented) - static create( - state: TaskState, - storage: TaskStore, - logger: Logger_2, - ): TaskAgent; - // (undocumented) - get done(): boolean; - // (undocumented) - emitLog(message: string, metadata?: JsonObject): Promise; - // (undocumented) - getWorkspaceName(): Promise; - // (undocumented) - get secrets(): TaskSecrets | undefined; - // (undocumented) - get spec(): TaskSpec; -} - // @public export interface TaskBroker { // (undocumented) - claim(): Promise; + claim(): Promise; // (undocumented) dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; // (undocumented) @@ -412,9 +374,47 @@ export interface TaskBroker { vacuumTasks(timeoutS: { timeoutS: number }): Promise; } +// @public +export interface TaskContext { + // (undocumented) + complete(result: CompletedTaskState, metadata?: JsonValue): Promise; + // (undocumented) + done: boolean; + // (undocumented) + emitLog(message: string, metadata?: JsonValue): Promise; + // (undocumented) + getWorkspaceName(): Promise; + // (undocumented) + secrets?: TaskSecrets; + // (undocumented) + spec: TaskSpec; +} + // @public export type TaskEventType = 'completion' | 'log'; +// @public +export class TaskManager implements TaskContext { + // (undocumented) + complete(result: CompletedTaskState, metadata?: JsonObject): Promise; + // (undocumented) + static create( + state: TaskState, + storage: TaskStore, + logger: Logger_2, + ): TaskManager; + // (undocumented) + get done(): boolean; + // (undocumented) + emitLog(message: string, metadata?: JsonObject): Promise; + // (undocumented) + getWorkspaceName(): Promise; + // (undocumented) + get secrets(): TaskSecrets | undefined; + // (undocumented) + get spec(): TaskSpec; +} + // @public export type TaskSecrets = { token: string | undefined; @@ -525,7 +525,7 @@ export class TaskWorker { // (undocumented) static createWorker(options: CreateWorkerOptions): TaskWorker; // (undocumented) - runOneTask(task: Task): Promise; + runOneTask(task: TaskContext): Promise; // (undocumented) start(): void; } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts index 0a843f4ddf..d07ff29fae 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -22,7 +22,7 @@ import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner'; import { TemplateActionRegistry } from '../actions'; import { ScmIntegrations } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; -import { Task, TaskSpec } from './types'; +import { TaskContext, TaskSpec } from './types'; describe('DefaultWorkflowRunner', () => { const logger = getVoidLogger(); @@ -38,7 +38,7 @@ describe('DefaultWorkflowRunner', () => { }), ); - const createMockTaskWithSpec = (spec: TaskSpec): Task => ({ + const createMockTaskWithSpec = (spec: TaskSpec): TaskContext => ({ spec, complete: async () => {}, done: false, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts index 831ef98784..77d0a7be09 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import { - Task, + TaskContext, WorkflowRunner, WorkflowResponse, TaskSpecV1beta2, @@ -72,7 +72,7 @@ export class HandlebarsWorkflowRunner implements WorkflowRunner { this.handlebars.registerHelper('eq', (a, b) => a === b); } - async execute(task: Task): Promise { + async execute(task: TaskContext): Promise { if (!isValidTaskSpec(task.spec)) { throw new InputError(`Task spec is not a valid v1beta2 task spec`); } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts index aa46747c68..91e772dc4e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts @@ -21,7 +21,7 @@ import { ScmIntegrations } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { HandlebarsWorkflowRunner } from './HandlebarsWorkflowRunner'; -import { Task, TaskSpec } from './types'; +import { TaskContext, TaskSpec } from './types'; import { RepoSpec } from '../actions/builtin/publish/util'; describe('LegacyWorkflowRunner', () => { @@ -37,7 +37,7 @@ describe('LegacyWorkflowRunner', () => { }), ); - const createMockTaskWithSpec = (spec: TaskSpec): Task => ({ + const createMockTaskWithSpec = (spec: TaskSpec): TaskContext => ({ spec, complete: async () => {}, done: false, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index 5cceb16324..587a93e1f2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -15,7 +15,7 @@ */ import { ScmIntegrations } from '@backstage/integration'; import { - Task, + TaskContext, TaskSpec, TaskSpecV1beta3, TaskStep, @@ -52,7 +52,13 @@ const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta3 => { return taskSpec.apiVersion === 'scaffolder.backstage.io/v1beta3'; }; -const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { +const createStepLogger = ({ + task, + step, +}: { + task: TaskContext; + step: TaskStep; +}) => { const metadata = { stepId: step.id }; const taskLogger = winston.createLogger({ level: process.env.LOG_LEVEL || 'info', @@ -162,7 +168,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner { }); } - async execute(task: Task): Promise { + async execute(task: TaskContext): Promise { if (!isValidTaskSpec(task.spec)) { throw new InputError( 'Wrong template version executed with the workflow engine', diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts index c5d94c1324..76fb8e00f0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -17,7 +17,7 @@ import { getVoidLogger, DatabaseManager } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { DatabaseTaskStore } from './DatabaseTaskStore'; -import { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; +import { StorageTaskBroker, TaskManager } from './StorageTaskBroker'; import { TaskSecrets, TaskSpec, SerializedTaskEvent } from './types'; async function createStore(): Promise { @@ -48,7 +48,7 @@ describe('StorageTaskBroker', () => { it('should claim a dispatched work item', async () => { const broker = new StorageTaskBroker(storage, logger); await broker.dispatch({} as TaskSpec); - await expect(broker.claim()).resolves.toEqual(expect.any(TaskAgent)); + await expect(broker.claim()).resolves.toEqual(expect.any(TaskManager)); }); it('should wait for a dispatched work item', async () => { @@ -58,7 +58,7 @@ describe('StorageTaskBroker', () => { await expect(Promise.race([promise, 'waiting'])).resolves.toBe('waiting'); await broker.dispatch({} as TaskSpec); - await expect(promise).resolves.toEqual(expect.any(TaskAgent)); + await expect(promise).resolves.toEqual(expect.any(TaskManager)); }); it('should dispatch multiple items and claim them in order', async () => { @@ -70,9 +70,9 @@ describe('StorageTaskBroker', () => { const taskA = await broker.claim(); const taskB = await broker.claim(); const taskC = await broker.claim(); - await expect(taskA).toEqual(expect.any(TaskAgent)); - await expect(taskB).toEqual(expect.any(TaskAgent)); - await expect(taskC).toEqual(expect.any(TaskAgent)); + await expect(taskA).toEqual(expect.any(TaskManager)); + await expect(taskB).toEqual(expect.any(TaskManager)); + await expect(taskC).toEqual(expect.any(TaskManager)); await expect(taskA.spec.steps[0].id).toBe('a'); await expect(taskB.spec.steps[0].id).toBe('b'); await expect(taskC.spec.steps[0].id).toBe('c'); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 2f4387ca4a..7845ca994a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -18,7 +18,7 @@ import { assertError } from '@backstage/errors'; import { Logger } from 'winston'; import { CompletedTaskState, - Task, + TaskContext, TaskSecrets, TaskSpec, TaskStore, @@ -29,17 +29,17 @@ import { } from './types'; /** - * TaskAgent + * TaskManager * * @public */ -export class TaskAgent implements Task { +export class TaskManager implements TaskContext { private isDone = false; private heartbeatTimeoutId?: ReturnType; static create(state: TaskState, storage: TaskStore, logger: Logger) { - const agent = new TaskAgent(state, storage, logger); + const agent = new TaskManager(state, storage, logger); agent.startTimeout(); return agent; } @@ -135,11 +135,11 @@ export class StorageTaskBroker implements TaskBroker { ) {} private deferredDispatch = defer(); - async claim(): Promise { + async claim(): Promise { for (;;) { const pendingTask = await this.storage.claimTask(); if (pendingTask) { - return TaskAgent.create( + return TaskManager.create( { taskId: pendingTask.id, spec: pendingTask.spec, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index fcc215be99..d22bdc4e30 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Task, TaskBroker, WorkflowRunner } from './types'; +import { TaskContext, TaskBroker, WorkflowRunner } from './types'; import { HandlebarsWorkflowRunner } from './HandlebarsWorkflowRunner'; import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner'; import { Logger } from 'winston'; @@ -94,7 +94,7 @@ export class TaskWorker { })(); } - async runOneTask(task: Task) { + async runOneTask(task: TaskContext) { try { const { output } = task.spec.apiVersion === 'scaffolder.backstage.io/v1beta3' diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index 941c27ef0e..510c862dae 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -14,7 +14,7 @@ * limitations under the License. */ export { DatabaseTaskStore } from './DatabaseTaskStore'; -export { TaskAgent } from './StorageTaskBroker'; +export { TaskManager } from './StorageTaskBroker'; export type { TaskState } from './StorageTaskBroker'; export { TaskWorker } from './TaskWorker'; export type { CreateWorkerOptions } from './TaskWorker'; @@ -31,7 +31,7 @@ export type { Status, TaskEventType, TaskBroker, - Task, + TaskContext, TaskStore, DispatchResult, } from './types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index f2e13923ec..6b3075844d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -139,7 +139,7 @@ export type DispatchResult = { * * @public */ -export interface Task { +export interface TaskContext { spec: TaskSpec; secrets?: TaskSecrets; done: boolean; @@ -154,7 +154,7 @@ export interface Task { * @public */ export interface TaskBroker { - claim(): Promise; + claim(): Promise; dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; vacuumTasks(timeoutS: { timeoutS: number }): Promise; observe( @@ -221,5 +221,5 @@ export interface TaskStore { export type WorkflowResponse = { output: { [key: string]: JsonValue } }; export interface WorkflowRunner { - execute(task: Task): Promise; + execute(task: TaskContext): Promise; } From 2223b6de60bccd7e85048727762bfcdba08590d8 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 27 Oct 2021 10:54:32 +0100 Subject: [PATCH 11/58] adds a todo message to the router import Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/src/service/router.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 71b6d27b5e..1c5343d458 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -40,6 +40,12 @@ import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; +/** + * TODO: The following should import directly from the router file. + * Due to a circular dependency between this plugin and the + * plugin-scaffolder-backend-module-cookiecutter plugin, it results in an error: + * TypeError: _pluginscaffolderbackend.createTemplateAction is not a function + */ import { createRouter } from '../index'; const createCatalogClient = (templates: any[] = []) => From 3236071136feb402a2b6fa107677e90d4176860c Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 27 Oct 2021 12:46:47 +0100 Subject: [PATCH 12/58] change name of task worker builder function also makes it async Signed-off-by: Brian Fletcher --- .../src/scaffolder/tasks/TaskWorker.test.ts | 6 +++--- .../scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts | 2 +- plugins/scaffolder-backend/src/service/router.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index 3cb73abb85..f33b9182ce 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -82,7 +82,7 @@ describe('TaskWorker', () => { it('should call the legacy workflow runner when the apiVersion is not beta3', async () => { const broker = new StorageTaskBroker(storage, logger); - const taskWorker = TaskWorker.createWorker({ + const taskWorker = TaskWorker.create({ logger, workingDirectory, integrations, @@ -113,7 +113,7 @@ describe('TaskWorker', () => { it('should call the default workflow runner when the apiVersion is beta3', async () => { const broker = new StorageTaskBroker(storage, logger); - const taskWorker = TaskWorker.createWorker({ + const taskWorker = TaskWorker.create({ logger, workingDirectory, integrations, @@ -142,7 +142,7 @@ describe('TaskWorker', () => { }); const broker = new StorageTaskBroker(storage, logger); - const taskWorker = TaskWorker.createWorker({ + const taskWorker = TaskWorker.create({ logger, workingDirectory, integrations, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index d22bdc4e30..8d89735361 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -56,7 +56,7 @@ export type CreateWorkerOptions = { export class TaskWorker { private constructor(private readonly options: TaskWorkerOptions) {} - static createWorker(options: CreateWorkerOptions) { + static async create(options: CreateWorkerOptions): Promise { const { taskBroker, logger, diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 700f2c6e6f..231c998f8b 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -99,7 +99,7 @@ export async function createRouter( const workers = []; for (let i = 0; i < (taskWorkers || 1); i++) { - const worker = TaskWorker.createWorker({ + const worker = await TaskWorker.create({ taskBroker, actionRegistry, integrations, From ab6b7191c55a144dedf6b94448314bf4d672b882 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 27 Oct 2021 13:08:13 +0100 Subject: [PATCH 13/58] change test to await the task worker create Signed-off-by: Brian Fletcher --- .../src/scaffolder/tasks/TaskWorker.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index f33b9182ce..9b6baf3b84 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -82,7 +82,7 @@ describe('TaskWorker', () => { it('should call the legacy workflow runner when the apiVersion is not beta3', async () => { const broker = new StorageTaskBroker(storage, logger); - const taskWorker = TaskWorker.create({ + const taskWorker = await TaskWorker.create({ logger, workingDirectory, integrations, @@ -113,7 +113,7 @@ describe('TaskWorker', () => { it('should call the default workflow runner when the apiVersion is beta3', async () => { const broker = new StorageTaskBroker(storage, logger); - const taskWorker = TaskWorker.create({ + const taskWorker = await TaskWorker.create({ logger, workingDirectory, integrations, @@ -142,7 +142,7 @@ describe('TaskWorker', () => { }); const broker = new StorageTaskBroker(storage, logger); - const taskWorker = TaskWorker.create({ + const taskWorker = await TaskWorker.create({ logger, workingDirectory, integrations, From 93d0ea461258ffe09070ef7088cbf9baf2bee0bd Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 27 Oct 2021 13:18:12 +0100 Subject: [PATCH 14/58] add api reports changes Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/api-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 5441c4de6b..6f9822cd09 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -523,7 +523,7 @@ export type TaskStoreListEventsOptions = { // @public export class TaskWorker { // (undocumented) - static createWorker(options: CreateWorkerOptions): TaskWorker; + static create(options: CreateWorkerOptions): Promise; // (undocumented) runOneTask(task: TaskContext): Promise; // (undocumented) From 66b5024b7c1eb4a944d79271a30d007405941c08 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 28 Oct 2021 13:02:07 +0100 Subject: [PATCH 15/58] only create datastore if needed for the broker Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/src/service/router.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 231c998f8b..814b235bf2 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -89,12 +89,17 @@ export async function createRouter( const workingDirectory = await getWorkingDirectory(config, logger); const entityClient = new CatalogEntityClient(catalogClient); const integrations = ScmIntegrations.fromConfig(config); + let taskBroker: TaskBroker; + + if (!options.taskBroker) { + const databaseTaskStore = await DatabaseTaskStore.create({ + database: await database.getClient(), + }); + taskBroker = new StorageTaskBroker(databaseTaskStore, logger); + } else { + taskBroker = options.taskBroker; + } - const databaseTaskStore = await DatabaseTaskStore.create({ - database: await database.getClient(), - }); - const taskBroker = - options.taskBroker || new StorageTaskBroker(databaseTaskStore, logger); const actionRegistry = new TemplateActionRegistry(); const workers = []; From 91f8644f17651c5e780aff0f5f33b7c4d8d67e5d Mon Sep 17 00:00:00 2001 From: Xavier Serrano Date: Wed, 27 Oct 2021 19:15:13 +0200 Subject: [PATCH 16/58] Add auth token to KafkaBackendClient via IdentityApi Signed-off-by: Xavier Serrano Signed-off-by: Xavier Serrano --- .changeset/long-bugs-kiss.md | 5 +++++ plugins/kafka/src/api/KafkaBackendClient.ts | 11 +++++++++-- plugins/kafka/src/plugin.ts | 6 ++++-- 3 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 .changeset/long-bugs-kiss.md diff --git a/.changeset/long-bugs-kiss.md b/.changeset/long-bugs-kiss.md new file mode 100644 index 0000000000..49e0edae72 --- /dev/null +++ b/.changeset/long-bugs-kiss.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-kafka': patch +--- + +Use IdentityApi to provide Auth Token for KafkaBackendClient Api calls diff --git a/plugins/kafka/src/api/KafkaBackendClient.ts b/plugins/kafka/src/api/KafkaBackendClient.ts index 988939ff1a..aea5736aad 100644 --- a/plugins/kafka/src/api/KafkaBackendClient.ts +++ b/plugins/kafka/src/api/KafkaBackendClient.ts @@ -15,21 +15,28 @@ */ import { KafkaApi, ConsumerGroupOffsetsResponse } from './types'; -import { DiscoveryApi } from '@backstage/core-plugin-api'; +import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api'; export class KafkaBackendClient implements KafkaApi { private readonly discoveryApi: DiscoveryApi; + private readonly identityApi: IdentityApi; - constructor(options: { discoveryApi: DiscoveryApi }) { + constructor(options: { + discoveryApi: DiscoveryApi; + identityApi: IdentityApi; + }) { this.discoveryApi = options.discoveryApi; + this.identityApi = options.identityApi; } private async internalGet(path: string): Promise { const url = `${await this.discoveryApi.getBaseUrl('kafka')}${path}`; + const idToken = await this.identityApi.getIdToken(); const response = await fetch(url, { method: 'GET', headers: { 'Content-Type': 'application/json', + ...(idToken && { Authorization: `Bearer ${idToken}` }), }, }); diff --git a/plugins/kafka/src/plugin.ts b/plugins/kafka/src/plugin.ts index e389e5e27a..539b0a0fda 100644 --- a/plugins/kafka/src/plugin.ts +++ b/plugins/kafka/src/plugin.ts @@ -21,6 +21,7 @@ import { createRoutableExtension, createRouteRef, discoveryApiRef, + identityApiRef, } from '@backstage/core-plugin-api'; export const rootCatalogKafkaRouteRef = createRouteRef({ @@ -33,8 +34,9 @@ export const kafkaPlugin = createPlugin({ apis: [ createApiFactory({ api: kafkaApiRef, - deps: { discoveryApi: discoveryApiRef }, - factory: ({ discoveryApi }) => new KafkaBackendClient({ discoveryApi }), + deps: { discoveryApi: discoveryApiRef, identityApi: identityApiRef }, + factory: ({ discoveryApi, identityApi }) => + new KafkaBackendClient({ discoveryApi, identityApi }), }), ], routes: { From 419054c97a98bcb93097c8c5585d483f69e7542e Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 28 Oct 2021 10:50:35 +0200 Subject: [PATCH 17/58] Remove usage of deprecated Keyboard class Signed-off-by: Johan Haals --- .../HeaderActionMenu/HeaderActionMenu.test.tsx | 18 ++++++------------ packages/test-utils/src/testUtils/Keyboard.js | 2 +- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/packages/core-components/src/layout/HeaderActionMenu/HeaderActionMenu.test.tsx b/packages/core-components/src/layout/HeaderActionMenu/HeaderActionMenu.test.tsx index b1bd398d15..58785c5c90 100644 --- a/packages/core-components/src/layout/HeaderActionMenu/HeaderActionMenu.test.tsx +++ b/packages/core-components/src/layout/HeaderActionMenu/HeaderActionMenu.test.tsx @@ -15,13 +15,10 @@ */ import React from 'react'; -import { render, fireEvent } from '@testing-library/react'; -import { - wrapInTestApp, - Keyboard, - renderInTestApp, -} from '@backstage/test-utils'; +import { fireEvent } from '@testing-library/react'; +import { renderInTestApp } from '@backstage/test-utils'; import { HeaderActionMenu } from './HeaderActionMenu'; +import userEvent from '@testing-library/user-event'; describe('', () => { it('renders without any items and without exploding', async () => { @@ -89,16 +86,13 @@ describe('', () => { }); it('should close when hitting escape', async () => { - const rendered = render( - wrapInTestApp( - , - ), + const rendered = await renderInTestApp( + , ); - expect(rendered.container.getAttribute('aria-hidden')).toBeNull(); fireEvent.click(rendered.getByTestId('header-action-menu')); expect(rendered.container.getAttribute('aria-hidden')).toBe('true'); - await Keyboard.type(rendered, ''); + userEvent.type(rendered.getByTestId('header-action-menu'), '{esc}'); expect(rendered.container.getAttribute('aria-hidden')).toBeNull(); }); }); diff --git a/packages/test-utils/src/testUtils/Keyboard.js b/packages/test-utils/src/testUtils/Keyboard.js index 3f4724d562..0d46d26363 100644 --- a/packages/test-utils/src/testUtils/Keyboard.js +++ b/packages/test-utils/src/testUtils/Keyboard.js @@ -25,7 +25,7 @@ const codes = { /** * @public - * @deprecated because it has no usages. Perhaps resurfaced in the future when need be. + * @deprecated superseded by @testing-library/user-event */ export class Keyboard { static async type(target, input) { From f808392183bbe32adf2c61fa4dba8ba85c4e8f8a Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 28 Oct 2021 13:38:41 +0200 Subject: [PATCH 18/58] reference library function Signed-off-by: Johan Haals --- packages/test-utils/src/testUtils/Keyboard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test-utils/src/testUtils/Keyboard.js b/packages/test-utils/src/testUtils/Keyboard.js index 0d46d26363..2a7928f8cd 100644 --- a/packages/test-utils/src/testUtils/Keyboard.js +++ b/packages/test-utils/src/testUtils/Keyboard.js @@ -25,7 +25,7 @@ const codes = { /** * @public - * @deprecated superseded by @testing-library/user-event + * @deprecated superseded by {@link @testing-library/user-event#userEvent} */ export class Keyboard { static async type(target, input) { From 71fd5cd735fe79a3182ae9de4a8b6f2f33918804 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 28 Oct 2021 14:58:08 +0200 Subject: [PATCH 19/58] add changeset Signed-off-by: Johan Haals --- .changeset/loud-bats-flow.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/loud-bats-flow.md diff --git a/.changeset/loud-bats-flow.md b/.changeset/loud-bats-flow.md new file mode 100644 index 0000000000..af4837dbbe --- /dev/null +++ b/.changeset/loud-bats-flow.md @@ -0,0 +1,5 @@ +--- +'@backstage/test-utils': patch +--- + +Update Keyboard deprecation with a link to the recommended successor From 8796c00de828f81c478f2fcf0a66b6530a00804a Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Thu, 28 Oct 2021 15:09:51 +0100 Subject: [PATCH 20/58] change return object for broker observe function Signed-off-by: Brian Fletcher --- plugins/scaffolder-backend/api-report.md | 4 +++- .../src/scaffolder/tasks/StorageTaskBroker.ts | 4 ++-- plugins/scaffolder-backend/src/scaffolder/tasks/types.ts | 2 +- plugins/scaffolder-backend/src/service/router.ts | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 6f9822cd09..3f9b77a3a3 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -369,7 +369,9 @@ export interface TaskBroker { events: SerializedTaskEvent[]; }, ) => void, - ): () => void; + ): { + unsubscribe: () => void; + }; // (undocumented) vacuumTasks(timeoutS: { timeoutS: number }): Promise; } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 7845ca994a..7e7ebe5436 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -178,7 +178,7 @@ export class StorageTaskBroker implements TaskBroker { error: Error | undefined, result: { events: SerializedTaskEvent[] }, ) => void, - ): () => void { + ): { unsubscribe: () => void } { const { taskId } = options; let cancelled = false; @@ -205,7 +205,7 @@ export class StorageTaskBroker implements TaskBroker { } })(); - return unsubscribe; + return { unsubscribe }; } async vacuumTasks(timeoutS: { timeoutS: number }): Promise { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 6b3075844d..6cfd914bd2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -166,7 +166,7 @@ export interface TaskBroker { error: Error | undefined, result: { events: SerializedTaskEvent[] }, ) => void, - ): () => void; + ): { unsubscribe: () => void }; get(taskId: string): Promise; } diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 814b235bf2..a78b04da78 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -258,7 +258,7 @@ export async function createRouter( }); // After client opens connection send all events as string - const unsubscribe = taskBroker.observe( + const { unsubscribe } = taskBroker.observe( { taskId, after }, (error, { events }) => { if (error) { From b45607a2ec899c7afba39f7d5987f1e064651b5a Mon Sep 17 00:00:00 2001 From: Andrew Thauer Date: Thu, 28 Oct 2021 17:56:59 -0400 Subject: [PATCH 21/58] fix: make techdocs s3 credentials optional in config schema Signed-off-by: Andrew Thauer --- .changeset/bright-taxis-stare.md | 5 +++++ plugins/techdocs-backend/config.d.ts | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 .changeset/bright-taxis-stare.md diff --git a/.changeset/bright-taxis-stare.md b/.changeset/bright-taxis-stare.md new file mode 100644 index 0000000000..e43c9b038b --- /dev/null +++ b/.changeset/bright-taxis-stare.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs-backend': patch +--- + +Make techdocs s3 publisher credentials config schema optional. diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts index ac31474fa9..0023a4b757 100644 --- a/plugins/techdocs-backend/config.d.ts +++ b/plugins/techdocs-backend/config.d.ts @@ -83,12 +83,12 @@ export interface Config { * User access key id * @visibility secret */ - accessKeyId: string; + accessKeyId?: string; /** * User secret access key * @visibility secret */ - secretAccessKey: string; + secretAccessKey?: string; /** * ARN of role to be assumed * @visibility backend From 4316c9632eb8dd65389bc9ab5ee01c91569fa4fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Oct 2021 04:09:55 +0000 Subject: [PATCH 22/58] build(deps): bump @azure/msal-node from 1.2.0 to 1.3.2 Bumps [@azure/msal-node](https://github.com/AzureAD/microsoft-authentication-library-for-js) from 1.2.0 to 1.3.2. - [Release notes](https://github.com/AzureAD/microsoft-authentication-library-for-js/releases) - [Commits](https://github.com/AzureAD/microsoft-authentication-library-for-js/compare/v1.2.0...v1.3.2) --- updated-dependencies: - dependency-name: "@azure/msal-node" dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- yarn.lock | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/yarn.lock b/yarn.lock index 517d66ba0c..2531e2b221 100644 --- a/yarn.lock +++ b/yarn.lock @@ -266,13 +266,20 @@ dependencies: tslib "^2.0.0" -"@azure/msal-common@^4.0.0", "@azure/msal-common@^4.4.0": +"@azure/msal-common@^4.0.0": version "4.4.0" resolved "https://registry.npmjs.org/@azure/msal-common/-/msal-common-4.4.0.tgz#818526042f78838ebc332fb735e7de64d8bccb45" integrity sha512-Qrs33Ctt2KM7NxArFPIUKc8UbIcm7zYxJFdJeQ9k7HKBhVk3e88CUz1Mw33cS/Jr+YA1H02OAzHg++bJ+4SFyQ== dependencies: debug "^4.1.1" +"@azure/msal-common@^5.0.1": + version "5.0.1" + resolved "https://registry.npmjs.org/@azure/msal-common/-/msal-common-5.0.1.tgz#234030b340cc63575e84190b96a8a336b69c7698" + integrity sha512-CmPR3XM9+CGUu7V/+bAwDxyN6XqWJJhVLmv7utT3sbgay4l5roVXsD1t4wURTs8PwzxmmnJOrhvvGhoDxUW69g== + dependencies: + debug "^4.1.1" + "@azure/msal-node@1.0.0-beta.6": version "1.0.0-beta.6" resolved "https://registry.npmjs.org/@azure/msal-node/-/msal-node-1.0.0-beta.6.tgz#da6bc3a3a861057c85586055960e069f162548ee" @@ -284,12 +291,12 @@ uuid "^8.3.0" "@azure/msal-node@^1.1.0": - version "1.2.0" - resolved "https://registry.npmjs.org/@azure/msal-node/-/msal-node-1.2.0.tgz#d08a5fb3391436715cb37c6eb44816013bdfa11e" - integrity sha512-79o5n483vslc7Qegh9+0BsxODRmlk6YYjVdl9jvwmAuF+i+oylq57e7RVhTVocKCbLCIMOKARI14JyKdDbW0WA== + version "1.3.2" + resolved "https://registry.npmjs.org/@azure/msal-node/-/msal-node-1.3.2.tgz#17665397c04b9cad57b42eb6e21d5d6f35d9b83a" + integrity sha512-aKU2lVRKhZa1IJ/Za/Ir6qlythQ3FHz0g0px3SbM4iC1otyr3ANS4mIn/6fmkpZDIHc8eAgJh2KMep1Yn2zpig== dependencies: - "@azure/msal-common" "^4.4.0" - axios "^0.21.1" + "@azure/msal-common" "^5.0.1" + axios "^0.21.4" jsonwebtoken "^8.5.1" uuid "^8.3.0" @@ -9348,7 +9355,7 @@ axios-cached-dns-resolve@0.5.2: pino "^5.12.2" pino-pretty "^2.6.0" -axios@^0.21.1: +axios@^0.21.1, axios@^0.21.4: version "0.21.4" resolved "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== From 547b902b68909cdfdbe1f9d403fa79828727e6cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Oct 2021 04:12:33 +0000 Subject: [PATCH 23/58] build(deps): bump @roadiehq/backstage-plugin-github-insights Bumps [@roadiehq/backstage-plugin-github-insights](https://github.com/RoadieHQ/roadie-backstage-plugins/tree/HEAD/plugins/frontend/backstage-plugin-github-insights) from 1.1.23 to 1.2.2. - [Release notes](https://github.com/RoadieHQ/roadie-backstage-plugins/releases) - [Changelog](https://github.com/RoadieHQ/roadie-backstage-plugins/blob/main/plugins/frontend/backstage-plugin-github-insights/docs/releases-widget.png) - [Commits](https://github.com/RoadieHQ/roadie-backstage-plugins/commits/HEAD/plugins/frontend/backstage-plugin-github-insights) --- updated-dependencies: - dependency-name: "@roadiehq/backstage-plugin-github-insights" dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- yarn.lock | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/yarn.lock b/yarn.lock index 517d66ba0c..914dc2c3a0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5184,15 +5184,16 @@ react-use "^17.2.4" "@roadiehq/backstage-plugin-github-insights@^1.1.23": - version "1.1.23" - resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-insights/-/backstage-plugin-github-insights-1.1.23.tgz#bcf7424df5a659df39d5aa75f5fffd8e02c9437e" - integrity sha512-3HL9nEtlaE+gFmV0xkrtZGbWHbS26z+NHzEk2KIyV8oqReQ+57PRLTVcXCjFA+djkKfP2hpu5s7zp72aVb0v+w== + version "1.2.2" + resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-insights/-/backstage-plugin-github-insights-1.2.2.tgz#09d958ed15adbb598afda34187f45deedec2264d" + integrity sha512-ybomCU/3NIQJYahp/E78wl+JuzpwkRkhyeCcySLA45GaCz/mEqAGSs4xDZancpV2ls6nxV3mKUw/DL9afmMGAw== dependencies: "@backstage/catalog-model" "^0.9.0" "@backstage/core-app-api" "^0.1.3" - "@backstage/core-components" "^0.3.0" + "@backstage/core-components" "^0.7.0" "@backstage/core-plugin-api" "^0.1.3" - "@backstage/plugin-catalog-react" "^0.4.0" + "@backstage/integration-react" "^0.1.10" + "@backstage/plugin-catalog-react" "^0.6.0" "@backstage/theme" "^0.2.7" "@date-io/core" "2.10.7" "@material-ui/core" "^4.11.0" From 2406f82644610745be3e3b8a15e5d6e09f3a59a8 Mon Sep 17 00:00:00 2001 From: djamaile Date: Fri, 29 Oct 2021 10:32:04 +0200 Subject: [PATCH 24/58] docs(#7768): include dependsOn to depict a relation between a component and resource Signed-off-by: djamaile --- docs/features/software-catalog/descriptor-format.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index c174b94557..7100e86ed0 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -502,6 +502,8 @@ spec: lifecycle: production owner: artist-relations-team system: artist-engagement-portal + dependsOn: + - resource:default/artists-db providesApis: - artist-api ``` From 4eb087ccdcce9d81786e0ca4b2433f21f8d8b5a8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 29 Oct 2021 11:03:53 +0200 Subject: [PATCH 25/58] techdocs-backend: release schema hotfix Signed-off-by: Patrik Oldsberg --- .changeset/bright-taxis-stare.md | 5 ----- packages/create-app/CHANGELOG.md | 2 ++ packages/create-app/package.json | 2 +- plugins/techdocs-backend/CHANGELOG.md | 6 ++++++ plugins/techdocs-backend/package.json | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) delete mode 100644 .changeset/bright-taxis-stare.md diff --git a/.changeset/bright-taxis-stare.md b/.changeset/bright-taxis-stare.md deleted file mode 100644 index e43c9b038b..0000000000 --- a/.changeset/bright-taxis-stare.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-techdocs-backend': patch ---- - -Make techdocs s3 publisher credentials config schema optional. diff --git a/packages/create-app/CHANGELOG.md b/packages/create-app/CHANGELOG.md index 05e067ce2c..95d99d0532 100644 --- a/packages/create-app/CHANGELOG.md +++ b/packages/create-app/CHANGELOG.md @@ -1,5 +1,7 @@ # @backstage/create-app +## 0.4.2 + ## 0.4.1 ### Patch Changes diff --git a/packages/create-app/package.json b/packages/create-app/package.json index 725f865435..ad4ab8dabd 100644 --- a/packages/create-app/package.json +++ b/packages/create-app/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/create-app", "description": "A CLI that helps you create your own Backstage app", - "version": "0.4.1", + "version": "0.4.2", "private": false, "publishConfig": { "access": "public" diff --git a/plugins/techdocs-backend/CHANGELOG.md b/plugins/techdocs-backend/CHANGELOG.md index 103306f79b..7331ea0147 100644 --- a/plugins/techdocs-backend/CHANGELOG.md +++ b/plugins/techdocs-backend/CHANGELOG.md @@ -1,5 +1,11 @@ # @backstage/plugin-techdocs-backend +## 0.10.7 + +### Patch Changes + +- b45607a2ec: Make techdocs s3 publisher credentials config schema optional. + ## 0.10.6 ### Patch Changes diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 3208b451fa..c71833948b 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/plugin-techdocs-backend", "description": "The Backstage backend plugin that renders technical documentation for your components", - "version": "0.10.6", + "version": "0.10.7", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", From 6129c89a47ef0df51b58708b2f0ad4604bcc2f6b Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 29 Oct 2021 11:48:39 +0200 Subject: [PATCH 26/58] Bump TechDocs container to v0.3.5. Signed-off-by: Eric Peterson --- .changeset/techdocs-buh-bump-ts.md | 5 +++++ packages/techdocs-common/api-report.md | 2 +- packages/techdocs-common/src/stages/generate/techdocs.ts | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 .changeset/techdocs-buh-bump-ts.md diff --git a/.changeset/techdocs-buh-bump-ts.md b/.changeset/techdocs-buh-bump-ts.md new file mode 100644 index 0000000000..438843e04b --- /dev/null +++ b/.changeset/techdocs-buh-bump-ts.md @@ -0,0 +1,5 @@ +--- +'@backstage/techdocs-common': patch +--- + +Default TechDocs container used at docs generation-time is now [v0.3.5](https://github.com/backstage/techdocs-container/releases/tag/v0.3.5). diff --git a/packages/techdocs-common/api-report.md b/packages/techdocs-common/api-report.md index f5a00c25ea..61371525e5 100644 --- a/packages/techdocs-common/api-report.md +++ b/packages/techdocs-common/api-report.md @@ -256,7 +256,7 @@ export class TechdocsGenerator implements GeneratorBase { config: Config; scmIntegrations: ScmIntegrationRegistry; }); - static readonly defaultDockerImage = 'spotify/techdocs:v0.3.4'; + static readonly defaultDockerImage = 'spotify/techdocs:v0.3.5'; // (undocumented) static fromConfig( config: Config, diff --git a/packages/techdocs-common/src/stages/generate/techdocs.ts b/packages/techdocs-common/src/stages/generate/techdocs.ts index e87278c2d4..728cab5a81 100644 --- a/packages/techdocs-common/src/stages/generate/techdocs.ts +++ b/packages/techdocs-common/src/stages/generate/techdocs.ts @@ -44,7 +44,7 @@ export class TechdocsGenerator implements GeneratorBase { * The default docker image (and version) used to generate content. Public * and static so that techdocs-common consumers can use the same version. */ - public static readonly defaultDockerImage = 'spotify/techdocs:v0.3.4'; + public static readonly defaultDockerImage = 'spotify/techdocs:v0.3.5'; private readonly logger: Logger; private readonly containerRunner: ContainerRunner; private readonly options: GeneratorConfig; From 75fc6ac85bbc3a6f73ef0cac9838ea17f9e23591 Mon Sep 17 00:00:00 2001 From: Jussi Hallila Date: Tue, 14 Sep 2021 12:54:52 +0200 Subject: [PATCH 27/58] Implement Tech Insights backend * Add common types and interfaces for Tech Insights. Exposing components needed to use and modify tech-insights-backend module and to implement individual Fact Retrievers, Fact Checkers and their persistence options. * Implement a framework to run fact retrievers and store fact data into the database. Add migration scripts to create a new database for `tech_insights`. * Create a default implementation of a FactChecker enabling users to construct checks and run them and generate scorecards based on checks. * To be able to use tech insights in your application you need to implement `FactRetriever`s to retrieve and return data for facts and register them to the tech-insights-backend. If you want to use fact checking functionality, you need to create `check`s and register them to an implementation of a `FactChecker`. For more information see documentation on the README.md files of the respective packages. Signed-off-by: Jussi Hallila --- .changeset/bright-pandas-rush.md | 17 + packages/backend/package.json | 3 + packages/backend/src/index.ts | 5 + packages/backend/src/plugins/techInsights.ts | 47 +++ .../.eslintrc.js | 3 + .../CHANGELOG.md | 7 + .../README.md | 83 +++++ .../api-report.md | 126 +++++++ .../package.json | 52 +++ .../src/index.ts | 32 ++ .../src/service/CheckRegistry.ts | 59 ++++ .../JsonRulesEngineFactChecker.test.ts | 293 ++++++++++++++++ .../src/service/JsonRulesEngineFactChecker.ts | 324 ++++++++++++++++++ .../src/service/validation-schema.json | 208 +++++++++++ .../src/setupTests.ts | 17 + .../src/types.ts | 87 +++++ plugins/tech-insights-backend/.eslintrc.js | 11 + plugins/tech-insights-backend/CHANGELOG.md | 7 + plugins/tech-insights-backend/README.md | 211 ++++++++++++ plugins/tech-insights-backend/api-report.md | 76 ++++ .../migrations/202109061111_fact_schemas.js | 57 +++ .../migrations/202109061212_facts.js | 73 ++++ plugins/tech-insights-backend/package.json | 64 ++++ plugins/tech-insights-backend/src/index.ts | 26 ++ .../src/service/DefaultTechInsightsBuilder.ts | 149 ++++++++ .../service/fact/FactRetrieverEngine.test.ts | 152 ++++++++ .../src/service/fact/FactRetrieverEngine.ts | 135 ++++++++ .../src/service/fact/FactRetrieverRegistry.ts | 63 ++++ .../service/persistence/DatabaseManager.ts | 99 ++++++ .../persistence/TechInsightsDatabase.test.ts | 218 ++++++++++++ .../persistence/TechInsightsDatabase.ts | 188 ++++++++++ .../src/service/router.test.ts | 125 +++++++ .../src/service/router.ts | 136 ++++++++ .../tech-insights-backend/src/setupTests.ts | 17 + plugins/tech-insights-common/.eslintrc.js | 3 + plugins/tech-insights-common/CHANGELOG.md | 7 + plugins/tech-insights-common/README.md | 3 + plugins/tech-insights-common/api-report.md | 167 +++++++++ plugins/tech-insights-common/package.json | 45 +++ plugins/tech-insights-common/src/checks.ts | 159 +++++++++ plugins/tech-insights-common/src/facts.ts | 196 +++++++++++ .../tech-insights-common/src/index.test.ts | 24 ++ plugins/tech-insights-common/src/index.ts | 20 ++ .../tech-insights-common/src/persistence.ts | 79 +++++ plugins/tech-insights-common/src/responses.ts | 100 ++++++ .../tech-insights-common/src/setupTests.ts | 17 + yarn.lock | 78 ++++- 47 files changed, 4065 insertions(+), 3 deletions(-) create mode 100644 .changeset/bright-pandas-rush.md create mode 100644 packages/backend/src/plugins/techInsights.ts create mode 100644 plugins/tech-insights-backend-module-jsonfc/.eslintrc.js create mode 100644 plugins/tech-insights-backend-module-jsonfc/CHANGELOG.md create mode 100644 plugins/tech-insights-backend-module-jsonfc/README.md create mode 100644 plugins/tech-insights-backend-module-jsonfc/api-report.md create mode 100644 plugins/tech-insights-backend-module-jsonfc/package.json create mode 100644 plugins/tech-insights-backend-module-jsonfc/src/index.ts create mode 100644 plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts create mode 100644 plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts create mode 100644 plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts create mode 100644 plugins/tech-insights-backend-module-jsonfc/src/service/validation-schema.json create mode 100644 plugins/tech-insights-backend-module-jsonfc/src/setupTests.ts create mode 100644 plugins/tech-insights-backend-module-jsonfc/src/types.ts create mode 100644 plugins/tech-insights-backend/.eslintrc.js create mode 100644 plugins/tech-insights-backend/CHANGELOG.md create mode 100644 plugins/tech-insights-backend/README.md create mode 100644 plugins/tech-insights-backend/api-report.md create mode 100644 plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js create mode 100644 plugins/tech-insights-backend/migrations/202109061212_facts.js create mode 100644 plugins/tech-insights-backend/package.json create mode 100644 plugins/tech-insights-backend/src/index.ts create mode 100644 plugins/tech-insights-backend/src/service/DefaultTechInsightsBuilder.ts create mode 100644 plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts create mode 100644 plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts create mode 100644 plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts create mode 100644 plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts create mode 100644 plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts create mode 100644 plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts create mode 100644 plugins/tech-insights-backend/src/service/router.test.ts create mode 100644 plugins/tech-insights-backend/src/service/router.ts create mode 100644 plugins/tech-insights-backend/src/setupTests.ts create mode 100644 plugins/tech-insights-common/.eslintrc.js create mode 100644 plugins/tech-insights-common/CHANGELOG.md create mode 100644 plugins/tech-insights-common/README.md create mode 100644 plugins/tech-insights-common/api-report.md create mode 100644 plugins/tech-insights-common/package.json create mode 100644 plugins/tech-insights-common/src/checks.ts create mode 100644 plugins/tech-insights-common/src/facts.ts create mode 100644 plugins/tech-insights-common/src/index.test.ts create mode 100644 plugins/tech-insights-common/src/index.ts create mode 100644 plugins/tech-insights-common/src/persistence.ts create mode 100644 plugins/tech-insights-common/src/responses.ts create mode 100644 plugins/tech-insights-common/src/setupTests.ts diff --git a/.changeset/bright-pandas-rush.md b/.changeset/bright-pandas-rush.md new file mode 100644 index 0000000000..86b3b819ee --- /dev/null +++ b/.changeset/bright-pandas-rush.md @@ -0,0 +1,17 @@ +--- +'@backstage/plugin-tech-insights-backend': minor +'@backstage/plugin-tech-insights-backend-module-jsonfc': minor +'@backstage/plugin-tech-insights-common': minor +--- + +## Add initial implementation of Tech Insights backend + +Add common types and interfaces for Tech Insights. Exposing components needed to use and modify tech-insights-backend module and to implement individual Fact Retrievers, Fact Checkers and their persistence options. + +Implement a framework to run fact retrievers and store fact data into the database. Add migration scripts to create a new database for `tech_insights`. + +Create a default implementation of a FactChecker enabling users to construct checks and run them and generate scorecards based on checks. + +To be able to use tech insights in your application you need to implement `FactRetriever`s to retrieve and return data for facts and register them to the tech-insights-backend. If you want to use fact checking functionality, you need to create `check`s and register them to an implementation of a `FactChecker`. + +For more information see documentation on the README.md files of the respective packages. diff --git a/packages/backend/package.json b/packages/backend/package.json index b9a28c2853..a56b302954 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -48,6 +48,9 @@ "@backstage/plugin-search-backend-module-elasticsearch": "^0.0.4", "@backstage/plugin-search-backend-module-pg": "^0.2.1", "@backstage/plugin-techdocs-backend": "^0.10.5", + "@backstage/plugin-tech-insights-backend": "^0.1.0", + "@backstage/plugin-tech-insights-common": "^0.1.0", + "@backstage/plugin-tech-insights-backend-module-jsonfc": "^0.1.0", "@backstage/plugin-todo-backend": "^0.1.13", "@gitbeaker/node": "^30.2.0", "@octokit/rest": "^18.5.3", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index b6c148dfba..6d40072f99 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -53,6 +53,7 @@ import graphql from './plugins/graphql'; import app from './plugins/app'; import badges from './plugins/badges'; import jenkins from './plugins/jenkins'; +import techInsights from './plugins/techInsights'; import { PluginEnvironment } from './types'; function makeCreateEnv(config: Config) { @@ -107,12 +108,16 @@ async function main() { const appEnv = useHotMemoize(module, () => createEnv('app')); const badgesEnv = useHotMemoize(module, () => createEnv('badges')); const jenkinsEnv = useHotMemoize(module, () => createEnv('jenkins')); + const techInsightsEnv = useHotMemoize(module, () => + createEnv('tech_insights'), + ); const apiRouter = Router(); apiRouter.use('/catalog', await catalog(catalogEnv)); apiRouter.use('/code-coverage', await codeCoverage(codeCoverageEnv)); apiRouter.use('/rollbar', await rollbar(rollbarEnv)); apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)); + apiRouter.use('/tech-insights', await techInsights(techInsightsEnv)); apiRouter.use('/auth', await auth(authEnv)); apiRouter.use('/azure-devops', await azureDevOps(azureDevOpsEnv)); apiRouter.use('/search', await search(searchEnv)); diff --git a/packages/backend/src/plugins/techInsights.ts b/packages/backend/src/plugins/techInsights.ts new file mode 100644 index 0000000000..c396f4b82b --- /dev/null +++ b/packages/backend/src/plugins/techInsights.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + createRouter, + DefaultTechInsightsBuilder, +} from '@backstage/plugin-tech-insights-backend'; +import { Router } from 'express'; +import { PluginEnvironment } from '../types'; +import { JsonRulesEngineFactCheckerFactory } from '@backstage/plugin-tech-insights-backend-module-jsonfc'; + +export default async function createPlugin({ + logger, + config, + discovery, + database, +}: PluginEnvironment): Promise { + const builder = new DefaultTechInsightsBuilder({ + logger, + config, + database, + discovery, + factRetrievers: [], + factCheckerFactory: new JsonRulesEngineFactCheckerFactory({ + checks: [], + logger, + }), + }); + + return await createRouter({ + ...(await builder.build()), + logger, + config, + }); +} diff --git a/plugins/tech-insights-backend-module-jsonfc/.eslintrc.js b/plugins/tech-insights-backend-module-jsonfc/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/tech-insights-backend-module-jsonfc/CHANGELOG.md b/plugins/tech-insights-backend-module-jsonfc/CHANGELOG.md new file mode 100644 index 0000000000..d0aced5bf5 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/CHANGELOG.md @@ -0,0 +1,7 @@ +# @backstage/plugin-tech-insights-backend-module-jsonfc + +## 0.0.1 + +### Patch Changes + +- Initial implementation diff --git a/plugins/tech-insights-backend-module-jsonfc/README.md b/plugins/tech-insights-backend-module-jsonfc/README.md new file mode 100644 index 0000000000..ea45a1b4b0 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/README.md @@ -0,0 +1,83 @@ +# Tech Insights Backend JSON Rules engine fact checker module + +This is an extension to module to tech-insights-backend plugin, which provides basic framework and functionality to implement tech insights within Backstage. + +This module provides functionality to run checks against a `json-rules-engine` and provide boolean logic by simply building checks using JSON conditions. + +## Getting started + +To add this FactChecker into your Tech Insights you need to install the module into your backend application: + +```bash +# From your Backstage root directory +cd packages/backend +yarn add @backstage/plugin-tech-insights-backend-module-jsonfc +``` + +and modify the `techInsights.ts` file to contain a reference to the FactCheckers implementation. + +```diff ++ import { JsonRulesEngineFactCheckerFactory } from '@backstage/plugin-tech-insights-backend-module-jsonfc'; + ++ const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ ++ checks: [], ++ logger, ++ }), + +const builder = new DefaultTechInsightsBuilder({ +logger, +config, +database, +discovery, +factRetrievers: [myFactRetrieverRegistration], ++ factCheckerFactory: myFactCheckerFactory +}); +``` + +By default this implementation comes with an in-memory storage to store checks. You can inject an additional data store by adding an implementation of `TechInsightCheckRegistry` into the constructor options when creating a `JsonRulesEngineFactCheckerFactory`. That can be done as follows + +```diff +const myTechInsightCheckRegistry: TechInsightCheckRegistry = // snip +const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ + checks: [], + logger, ++ checkRegistry: myTechInsightCheckRegistry +}), + +``` + +## Adding checks + +Checks for this FactChecker are constructed as `json-rules-engine` compatible JSON rules. A check could look like the following for example: + +```ts +import { TechInsightJsonRuleCheck } from '../types'; + +export const exampleCheck: TechInsightJsonRuleCheck = { + id: 'demodatacheck', // Unique identifier of this check + name: 'demodatacheck', // A human readable name of this check to be displayed in the UI + description: 'A fact check for demoing purposes', // A description to be displayed in the UI + factRefs: ['demo-poc.factretriever'], // References to fact containers that this check uses. See documentation on FactRetrievers for more information on these + rule: { + // The actual rule + conditions: { + all: [ + // 2 options are available, all and any conditions. + { + fact: 'examplenumberfact', // Reference to an individual fact to check against + operator: 'greaterThanInclusive', // Operator to use. See: https://github.com/CacheControl/json-rules-engine/blob/master/docs/rules.md#operators for more + value: 2, // The threshold value that the fact must satisfy + }, + ], + }, + }, + successMetadata: { + // Additional metadata to be returned if the check has passed + link: 'https://link.to.some.information.com', + }, + failureMetadata: { + // Additional metadata to be returned if the check has failed + link: 'https://sonar.mysonarqube.com/increasing-number-value', + }, +}; +``` diff --git a/plugins/tech-insights-backend-module-jsonfc/api-report.md b/plugins/tech-insights-backend-module-jsonfc/api-report.md new file mode 100644 index 0000000000..bf08028a00 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/api-report.md @@ -0,0 +1,126 @@ +## API Report File for "@backstage/plugin-tech-insights-backend-module-jsonfc" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BooleanCheckResult } from '@backstage/plugin-tech-insights-common'; +import { CheckResponse } from '@backstage/plugin-tech-insights-common'; +import { DynamicFactCallback } from 'json-rules-engine'; +import { FactChecker } from '@backstage/plugin-tech-insights-common'; +import { FactOptions } from 'json-rules-engine'; +import { Logger as Logger_2 } from 'winston'; +import { TechInsightCheck } from '@backstage/plugin-tech-insights-common'; +import { TechInsightCheckRegistry } from '@backstage/plugin-tech-insights-common'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-common'; +import { TopLevelCondition } from 'json-rules-engine'; + +// @public (undocumented) +export type CheckCondition = { + operator: string; + fact: string; + factValue: any; + factResult: any; + result: boolean; +}; + +// @public (undocumented) +export interface DynamicFact { + // (undocumented) + calculationMethod: DynamicFactCallback | T; + // (undocumented) + id: string; + // (undocumented) + options?: FactOptions; +} + +// @public (undocumented) +export interface JsonRuleBooleanCheckResult extends BooleanCheckResult { + // (undocumented) + check: JsonRuleCheckResponse; +} + +// @public (undocumented) +export interface JsonRuleCheckResponse extends CheckResponse { + // (undocumented) + rule: { + conditions: ResponseTopLevelCondition & { + priority: number; + }; + }; +} + +// @public +export class JsonRulesEngineFactChecker + implements FactChecker +{ + constructor({ + checks, + repository, + logger, + checkRegistry, + }: JsonRulesEngineFactCheckerOptions); + // (undocumented) + addCheck(check: TechInsightJsonRuleCheck): Promise; + // (undocumented) + getChecks(): Promise; + // (undocumented) + runChecks( + entity: string, + checks: string[], + ): Promise; + // (undocumented) + validate(check: TechInsightJsonRuleCheck): Promise; +} + +// @public +export class JsonRulesEngineFactCheckerFactory { + constructor({ + checks, + logger, + checkRegistry, + }: JsonRulesEngineFactCheckerFactoryOptions); + // (undocumented) + construct(repository: TechInsightsStore): JsonRulesEngineFactChecker; +} + +// @public +export type JsonRulesEngineFactCheckerFactoryOptions = { + checks: TechInsightJsonRuleCheck[]; + logger: Logger_2; + checkRegistry?: TechInsightCheckRegistry; +}; + +// @public +export type JsonRulesEngineFactCheckerOptions = { + checks: TechInsightJsonRuleCheck[]; + repository: TechInsightsStore; + logger: Logger_2; + checkRegistry?: TechInsightCheckRegistry; +}; + +// @public (undocumented) +export type ResponseTopLevelCondition = + | { + all: CheckCondition[]; + } + | { + any: CheckCondition[]; + }; + +// @public (undocumented) +export type Rule = { + conditions: TopLevelCondition; + name?: string; + priority?: number; +}; + +// @public (undocumented) +export interface TechInsightJsonRuleCheck extends TechInsightCheck { + // (undocumented) + dynamicFacts?: DynamicFact[]; + // (undocumented) + rule: Rule; +} + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/tech-insights-backend-module-jsonfc/package.json b/plugins/tech-insights-backend-module-jsonfc/package.json new file mode 100644 index 0000000000..945947dd3b --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/package.json @@ -0,0 +1,52 @@ +{ + "name": "@backstage/plugin-tech-insights-backend-module-jsonfc", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/tech-insights-backend-module-jsonfc" + }, + "keywords": [ + "backstage", + "tech-insights", + "scorecard" + ], + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/backend-common": "^0.9.0", + "@backstage/config": "^0.1.8", + "@backstage/errors": "^0.1.1", + "@backstage/plugin-tech-insights-common": "^0.1.0", + "ajv": "^7.0.3", + "json-rules-engine": "^6.1.2", + "lodash": "^4.17.21", + "luxon": "^2.0.2", + "node-cron": "^3.0.0", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/cli": "^0.7.1", + "@types/node-cron": "^2.0.4" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/tech-insights-backend-module-jsonfc/src/index.ts b/plugins/tech-insights-backend-module-jsonfc/src/index.ts new file mode 100644 index 0000000000..a2561eaa3b --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/index.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { + JsonRulesEngineFactCheckerFactory, + JsonRulesEngineFactChecker, +} from './service/JsonRulesEngineFactChecker'; +export type { + JsonRulesEngineFactCheckerFactoryOptions, + JsonRulesEngineFactCheckerOptions, +} from './service/JsonRulesEngineFactChecker'; +export type { + JsonRuleCheckResponse, + JsonRuleBooleanCheckResult, + TechInsightJsonRuleCheck, + ResponseTopLevelCondition, + Rule, + DynamicFact, + CheckCondition, +} from './types'; diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts new file mode 100644 index 0000000000..2ae1ac0b83 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts @@ -0,0 +1,59 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConflictError, NotFoundError } from '@backstage/errors'; +import { + TechInsightCheck, + TechInsightCheckRegistry, +} from '@backstage/plugin-tech-insights-common'; + +export class DefaultCheckRegistry + implements TechInsightCheckRegistry +{ + private readonly checks = new Map(); + + constructor(checks: CheckType[]) { + checks.forEach(check => { + this.register(check); + }); + } + + async register(check: CheckType) { + if (this.checks.has(check.id)) { + throw new ConflictError( + `Tech insight check with id ${check.id} has already been registered`, + ); + } + this.checks.set(check.id, check); + } + + async get(checkId: string): Promise { + const check = this.checks.get(checkId); + if (!check) { + throw new NotFoundError( + `Tech insight check with id '${checkId}' is not registered.`, + ); + } + return check; + } + async getAll(checks: string[]): Promise { + return Promise.all(checks.map(checkId => this.get(checkId))); + } + + async list(): Promise { + return [...this.checks.values()]; + } +} diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts new file mode 100644 index 0000000000..d574e13c5f --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts @@ -0,0 +1,293 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + TechInsightCheckRegistry, + TechInsightsStore, +} from '@backstage/plugin-tech-insights-common'; +import { JsonRulesEngineFactCheckerFactory } from '../index'; +import { getVoidLogger } from '@backstage/backend-common'; +import { TechInsightJsonRuleCheck } from '../types'; + +const testChecks: Record = { + broken: [ + { + id: 'brokenTestCheck', + name: 'brokenTestCheck', + description: 'Broken Check For Testing', + factRefs: ['test-factretriever'], + rule: { + conditions: { + all: [ + { + fact: 'testnumberfact', + operator: 'largerThan', + value: 1, + }, + ], + }, + }, + }, + ], + broken2: [ + { + id: 'brokenTestCheck2', + name: 'brokenTestCheck2', + description: 'Second Broken Check For Testing', + factRefs: ['non-existing-factretriever'], + rule: { + conditions: { + any: [ + { + fact: 'somefact', + operator: 'lessThan', + value: 1, + }, + ], + }, + }, + }, + ], + simple: [ + { + id: 'simpleTestCheck', + name: 'simpleTestCheck', + description: 'Simple Check For Testing', + factRefs: ['test-factretriever'], + rule: { + conditions: { + all: [ + { + fact: 'testnumberfact', + operator: 'lessThan', + value: 5, + }, + ], + }, + }, + }, + ], + + simple2: [ + { + id: 'simpleTestCheck2', + name: 'simpleTestCheck2', + description: 'Second Simple Check For Testing', + factRefs: ['test-factretriever'], + rule: { + conditions: { + all: [ + { + fact: 'testnumberfact', + operator: 'greaterThan', + value: 2, + }, + ], + }, + }, + }, + ], +}; + +const latestSchemasMock = jest.fn().mockImplementation(() => [ + { + version: '0.0.1', + id: 2, + ref: 'test-factretriever', + schema: { + testnumberfact: { + type: 'integer', + description: '', + entityKinds: ['component'], + }, + }, + }, +]); +const factsBetweenTimestampsForRefsMock = jest.fn(); +const latestFactsForRefsMock = jest.fn().mockImplementation(() => ({})); +const mockCheckRegistry = { + getAll(checks: string[]) { + return checks.flatMap(check => testChecks[check]); + }, +} as unknown as TechInsightCheckRegistry; + +const mockRepository: TechInsightsStore = { + getLatestFactsForRefs: latestFactsForRefsMock, + getFactsBetweenTimestampsForRefs: factsBetweenTimestampsForRefsMock, + getLatestSchemas: latestSchemasMock, +} as unknown as TechInsightsStore; + +describe('JsonRulesEngineFactChecker', () => { + const factChecker = new JsonRulesEngineFactCheckerFactory({ + checkRegistry: mockCheckRegistry, + checks: [], + logger: getVoidLogger(), + }).construct(mockRepository); + + describe('when running checks', () => { + it('should throw on incorrectly configured checks conditions', async () => { + const cur = async () => await factChecker.runChecks('a/a/a', ['broken']); + await expect(cur()).rejects.toThrowError( + 'Failed to run rules engine, Unknown operator: largerThan', + ); + }); + + it('should handle cases where wrong facts are referenced', async () => { + const cur = async () => await factChecker.runChecks('a/a/a', ['broken2']); + await expect(cur()).rejects.toThrowError( + 'Failed to run rules engine, Undefined fact: somefact', + ); + }); + it('should respond with result, facts, fact schemas and checks', async () => { + latestFactsForRefsMock.mockImplementation(() => + Promise.resolve({ + ['test-factretriever']: { + ref: 'test-factretriever', + facts: { + testnumberfact: 3, + }, + }, + }), + ); + const results = await factChecker.runChecks('a/a/a', ['simple']); + expect(results).toMatchObject([ + { + facts: { + testnumberfact: { + value: 3, + type: 'integer', + description: '', + entityKinds: ['component'], + }, + }, + result: true, + check: { + id: 'simpleTestCheck', + name: 'simpleTestCheck', + description: 'Simple Check For Testing', + factRefs: ['test-factretriever'], + rule: { + conditions: { + all: [ + { + fact: 'testnumberfact', + factResult: 3, + operator: 'lessThan', + result: true, + value: 5, + }, + ], + priority: 1, + }, + }, + }, + }, + ]); + }); + + it('should gracefully handle multiple check at once', async () => { + latestFactsForRefsMock.mockImplementation(() => + Promise.resolve({ + ['test-factretriever']: { + ref: 'test-factretriever', + facts: { + testnumberfact: 3, + }, + }, + }), + ); + const results = await factChecker.runChecks('a/a/a', [ + 'simple', + 'simple2', + ]); + expect(results).toMatchObject([ + { + facts: { + testnumberfact: { + value: 3, + type: 'integer', + description: '', + entityKinds: ['component'], + }, + }, + result: true, + check: { + id: 'simpleTestCheck', + name: 'simpleTestCheck', + description: 'Simple Check For Testing', + factRefs: ['test-factretriever'], + rule: { + conditions: { + priority: 1, + all: [ + { + operator: 'lessThan', + value: 5, + fact: 'testnumberfact', + factResult: 3, + result: true, + }, + ], + }, + }, + }, + }, + { + facts: { + testnumberfact: { + value: 3, + type: 'integer', + description: '', + entityKinds: ['component'], + }, + }, + result: true, + check: { + id: 'simpleTestCheck2', + name: 'simpleTestCheck2', + description: 'Second Simple Check For Testing', + factRefs: ['test-factretriever'], + rule: { + conditions: { + priority: 1, + all: [ + { + operator: 'greaterThan', + value: 2, + fact: 'testnumberfact', + factResult: 3, + result: true, + }, + ], + }, + }, + }, + }, + ]); + }); + }); + + describe('when validating checks', () => { + it('should succeed on valid rules', async () => { + const isValid = await factChecker.validate(testChecks.simple[0]); + expect(isValid).toBeTruthy(); + }); + it('should fail on broken rules', async () => { + const isValid = await factChecker.validate(testChecks.broken[0]); + expect(isValid).toBeFalsy(); + }); + }); +}); diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts new file mode 100644 index 0000000000..964575b494 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts @@ -0,0 +1,324 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonRuleBooleanCheckResult, TechInsightJsonRuleCheck } from '../types'; +import { + FactChecker, + FactResponse, + FactValueDefinitions, + TechInsightCheckRegistry, + FlatTechInsightFact, + TechInsightsStore, +} from '@backstage/plugin-tech-insights-common'; +import { Engine, EngineResult, TopLevelCondition } from 'json-rules-engine'; +import { DefaultCheckRegistry } from './CheckRegistry'; +import { Logger } from 'winston'; +import { pick } from 'lodash'; +import Ajv from 'ajv'; +import * as validationSchema from './validation-schema.json'; + +const noopEvent = { + type: 'noop', +}; + +/** + * @public + * Should actually be at-internal + * + * Constructor options for JsonRulesEngineFactChecker + */ +export type JsonRulesEngineFactCheckerOptions = { + checks: TechInsightJsonRuleCheck[]; + repository: TechInsightsStore; + logger: Logger; + checkRegistry?: TechInsightCheckRegistry; +}; + +/** + * @public + * Should actually be at-internal + * + * FactChecker implementation using json-rules-engine + */ +export class JsonRulesEngineFactChecker + implements FactChecker +{ + private readonly checkRegistry: TechInsightCheckRegistry; + private repository: TechInsightsStore; + private readonly logger: Logger; + + constructor({ + checks, + repository, + logger, + checkRegistry, + }: JsonRulesEngineFactCheckerOptions) { + this.repository = repository; + this.logger = logger; + checks.forEach(check => this.validate(check)); + this.checkRegistry = checkRegistry ?? new DefaultCheckRegistry(checks); + } + + async runChecks( + entity: string, + checks: string[], + ): Promise { + const engine = new Engine(); + const techInsightChecks = await this.checkRegistry.getAll(checks); + const factRefs = techInsightChecks.flatMap(it => it.factRefs); + const facts = await this.repository.getLatestFactsForRefs(factRefs, entity); + techInsightChecks.forEach(techInsightCheck => { + const rule = techInsightCheck.rule; + rule.name = techInsightCheck.id; + engine.addRule({ ...techInsightCheck.rule, event: noopEvent }); + + if (techInsightCheck.dynamicFacts) { + techInsightCheck.dynamicFacts.forEach(it => + engine.addFact(it.id, it.calculationMethod, it.options), + ); + } + }); + + const factValues = Object.values(facts).reduce( + (acc, it) => ({ ...acc, ...it.facts }), + {}, + ); + + try { + const results = await engine.run(factValues); + return await this.ruleEngineResultsToCheckResponse( + results, + techInsightChecks, + Object.values(facts), + ); + } catch (e) { + throw new Error(`Failed to run rules engine, ${e.message}`); + } + } + + async validate(check: TechInsightJsonRuleCheck): Promise { + const ajv = new Ajv({ verbose: true }); + const validator = ajv.compile(validationSchema); + const isValidToSchema = validator(check.rule); + if (!isValidToSchema) { + this.logger.warn( + 'Failed to to validate conditions against JSON schema', + validator.errors, + ); + return false; + } + + const existingSchemas = await this.repository.getLatestSchemas( + check.factRefs, + ); + const references = this.retrieveFactReferences(check.rule.conditions); + const results = references.map(ref => ({ + ref, + result: existingSchemas.some(schema => schema.schema.hasOwnProperty(ref)), + })); + const failedReferences = results.filter(it => !it.result); + failedReferences.forEach(it => { + this.logger.warn( + `Validation failed for check ${check.name}. Reference to value ${ + it.ref + } does not exists in referred fact schemas: ${check.factRefs.join( + ',', + )}`, + ); + }); + return failedReferences.length === 0; + } + + getChecks(): Promise { + return this.checkRegistry.list(); + } + + async addCheck(check: TechInsightJsonRuleCheck): Promise { + if (!(await this.validate(check))) { + this.logger.warn( + `Check validation failed when adding check ${check.name} to check registry.`, + ); + return false; + } + this.checkRegistry.register(check); + return true; + } + + private retrieveFactReferences( + condition: TopLevelCondition | { fact: string }, + ): string[] { + let results: string[] = []; + if ('all' in condition) { + results = results.concat( + condition.all.flatMap(con => this.retrieveFactReferences(con)), + ); + } else if ('any' in condition) { + results = results.concat( + condition.any.flatMap(con => this.retrieveFactReferences(con)), + ); + } else { + results.push(condition.fact); + } + return results; + } + + private async ruleEngineResultsToCheckResponse( + results: EngineResult, + techInsightChecks: TechInsightJsonRuleCheck[], + facts: FlatTechInsightFact[], + ) { + return await Promise.all( + [ + ...(results.results && results.results), + ...(results.failureResults && results.failureResults), + ].map(async result => { + const techInsightCheck = techInsightChecks.find( + check => check.id === result.name, + ); + if (!techInsightCheck) { + // This should never happen, we just constructed these based on each other + throw new Error( + `Failed to determine tech insight check with id ${result.name}. Discrepancy between ran rule engine and configured checks.`, + ); + } + const factResponse = await this.constructFactInformationResponse( + facts, + techInsightCheck, + ); + return { + facts: factResponse, + result: result.result, + check: JsonRulesEngineFactChecker.constructCheckResponse( + techInsightCheck, + result, + ), + }; + }), + ); + } + + private static constructCheckResponse( + techInsightCheck: TechInsightJsonRuleCheck, + result: any, + ) { + const returnable = { + id: techInsightCheck.id, + name: techInsightCheck.name, + description: techInsightCheck.description, + factRefs: techInsightCheck.factRefs, + metadata: result.result + ? techInsightCheck.successMetadata + : techInsightCheck.failureMetadata, + rule: { conditions: {} }, + }; + + if ('toJSON' in result) { + // Results serialize "wrong" since the objects are creating their own serialization implementations + // 'toJSON' should always be present in the result object but it is missing from the types + const rule = JSON.parse(result.toJSON()); + return { ...returnable, rule: pick(rule, ['conditions']) }; + } + return returnable; + } + + private async constructFactInformationResponse( + facts: FlatTechInsightFact[], + techInsightCheck: TechInsightJsonRuleCheck, + ): Promise { + const factSchemas = await this.repository.getLatestSchemas( + techInsightCheck.factRefs, + ); + const schemas: FactValueDefinitions = factSchemas.reduce( + (acc, schema) => ({ ...acc, ...schema.schema }), + {}, + ); + const individualFacts = this.retrieveFactReferences( + techInsightCheck.rule.conditions, + ); + const factValues = facts + .filter(factContainer => + techInsightCheck.factRefs.includes(factContainer.ref), + ) + .reduce( + (acc, factContainer) => ({ + ...acc, + ...pick(factContainer.facts, individualFacts), + }), + {}, + ); + return Object.entries(factValues).reduce((acc, [key, value]) => { + return { + ...acc, + [key]: { + value, + ...schemas[key], + }, + }; + }, {}); + } +} + +/** + * @public + * + * Constructor options for JsonRulesEngineFactCheckerFactory + * + * Implementation of checkRegistry is optional. + * If there is a need to use persistent storage for checks, it is recommended to inject a storage implementation here. + * Otherwise an in-memory option is instantiated and used. + */ +export type JsonRulesEngineFactCheckerFactoryOptions = { + checks: TechInsightJsonRuleCheck[]; + logger: Logger; + checkRegistry?: TechInsightCheckRegistry; +}; + +/** + * @public + * + * Factory to construct JsonRulesEngineFactChecker + * Can be constructed with optional implementation of CheckInsightCheckRegistry if needed. + * Otherwise defaults to using in-memory CheckRegistry + */ +export class JsonRulesEngineFactCheckerFactory { + private readonly checks: TechInsightJsonRuleCheck[]; + private readonly logger: Logger; + private readonly checkRegistry?: TechInsightCheckRegistry; + + constructor({ + checks, + logger, + checkRegistry, + }: JsonRulesEngineFactCheckerFactoryOptions) { + this.logger = logger; + this.checks = checks; + this.checkRegistry = checkRegistry; + } + + /** + * @param repository - Implementation of TechInsightsStore. Used by the returned JsonRulesEngineFactChecker + * to retrieve fact and fact schema data + * @returns JsonRulesEngineFactChecker implementation + */ + construct(repository: TechInsightsStore) { + return new JsonRulesEngineFactChecker({ + checks: this.checks, + logger: this.logger, + checkRegistry: this.checkRegistry, + repository, + }); + } +} diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/validation-schema.json b/plugins/tech-insights-backend-module-jsonfc/src/service/validation-schema.json new file mode 100644 index 0000000000..9020ba0735 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/validation-schema.json @@ -0,0 +1,208 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://github.com/CacheControl/json-rules-engine/rule-schema.json", + "type": "object", + "title": "Fact Checks", + "description": "Checks contain a set of conditions and a single event. When the engine is run, each check condition is evaluated and results returned", + "required": ["conditions"], + "properties": { + "conditions": { + "$ref": "#/definitions/conditions" + }, + "priority": { + "$id": "#/properties/priority", + "anyOf": [ + { + "type": "integer", + "minimum": 1 + } + ], + "title": "Priority", + "description": "Dictates when check should be run, relative to other check. Higher priority checks are run before lower priority checks. Checks with the same priority are run in parallel. Priority must be a positive, non-zero integer.", + "default": 1, + "examples": [1] + } + }, + "definitions": { + "conditions": { + "type": "object", + "title": "Conditions", + "description": "Check conditions are a combination of facts, operators, and values that determine whether the check is a success or a failure. The simplest form of a condition consists of a fact, an operator, and a value. When the engine runs, the operator is used to compare the fact against the value. Each check's conditions must have either an all or an any operator at its root, containing an array of conditions. The all operator specifies that all conditions contained within must be truthy for the check to be considered a success. The any operator only requires one condition to be truthy for the check to succeed.", + "default": {}, + "examples": [ + { + "all": [ + { + "value": true, + "fact": "displayMessage", + "operator": "equal" + } + ] + } + ], + "oneOf": [ + { + "required": ["any"] + }, + { + "required": ["all"] + } + ], + "properties": { + "any": { + "$ref": "#/definitions/conditionArray" + }, + "all": { + "$ref": "#/definitions/conditionArray" + } + } + }, + "conditionArray": { + "type": "array", + "title": "Condition Array", + "description": "An array of conditions with a possible recursive inclusion of another condition array.", + "default": [], + "items": { + "anyOf": [ + { + "$ref": "#/definitions/conditions" + }, + { + "$ref": "#/definitions/condition" + } + ] + } + }, + "condition": { + "type": "object", + "title": "Condition", + "description": "Check conditions are a combination of facts, operators, and values that determine whether the check is a success or a failure. The simplest form of a condition consists of a fact, an operator, and a value. When the engine runs, the operator is used to compare the fact against the value.", + "default": { + "fact": "my-fact", + "operator": "lessThanInclusive", + "value": 1 + }, + "examples": [ + { + "fact": "gameDuration", + "operator": "equal", + "value": 40.0 + }, + { + "value": 5.0, + "fact": "personalFoulCount", + "operator": "greaterThanInclusive" + }, + { + "fact": "product-price", + "operator": "greaterThan", + "path": "$.price", + "value": 100.0, + "params": { + "productId": "widget" + } + } + ], + "required": ["fact", "operator", "value"], + "properties": { + "fact": { + "type": "string", + "title": "Fact", + "description": "Facts are methods or constants registered with the engine prior to runtime and referenced within check conditions. Each fact method should be a pure function that may return a either computed value, or promise that resolves to a computed value. As check conditions are evaluated during runtime, they retrieve fact values dynamically and use the condition operator to compare the fact result with the condition value.", + "default": "", + "examples": ["gameDuration"] + }, + "operator": { + "type": "string", + "anyOf": [ + { + "const": "equal", + "title": "fact must equal value" + }, + { + "const": "notEqual", + "title": "fact must not equal value" + }, + { + "const": "lessThan", + "title": "fact must be less than value" + }, + { + "const": "lessThanInclusive", + "title": "fact must be less than or equal to value" + }, + { + "const": "greaterThan", + "title": "fact must be greater than value" + }, + { + "const": "greaterThanInclusive", + "title": "fact must be greater than or equal to value" + }, + { + "const": "in", + "title": "fact must be included in value (an array)" + }, + { + "const": "notIn", + "title": "fact must not be included in value (an array)" + }, + { + "const": "contains", + "title": "fact (an array) must include value" + }, + { + "const": "doesNotContain", + "title": "fact (an array) must not include value" + } + ], + "title": "Operator", + "description": "The operator compares the value returned by the fact to what is stored in the value property. If the result is truthy, the condition passes.", + "default": "", + "examples": ["equal"] + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object" + }, + { + "type": "array" + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ], + "title": "Value", + "description": "The value the fact should be compared to.", + "default": 0, + "examples": [40] + }, + "params": { + "type": "object", + "title": "Event Params", + "description": "Optional helper params to make available to the event processor.", + "default": {}, + "examples": [ + { + "customProperty": "customValue" + } + ] + }, + "path": { + "type": "string", + "title": "Path", + "description": "For more complex data structures, writing a separate fact handler for each object property quickly becomes verbose and unwieldy. To address this, a path property may be provided to traverse fact data using json-path syntax. Json-path support is provided by jsonpath-plus", + "default": "", + "examples": ["$.price"] + } + } + } + } +} diff --git a/plugins/tech-insights-backend-module-jsonfc/src/setupTests.ts b/plugins/tech-insights-backend-module-jsonfc/src/setupTests.ts new file mode 100644 index 0000000000..a330613afb --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/plugins/tech-insights-backend-module-jsonfc/src/types.ts b/plugins/tech-insights-backend-module-jsonfc/src/types.ts new file mode 100644 index 0000000000..5f6210d110 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/types.ts @@ -0,0 +1,87 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + DynamicFactCallback, + FactOptions, + TopLevelCondition, +} from 'json-rules-engine'; +import { + BooleanCheckResult, + CheckResponse, + TechInsightCheck, +} from '@backstage/plugin-tech-insights-common'; + +/** + * @public + */ +export interface DynamicFact { + id: string; + calculationMethod: DynamicFactCallback | T; + options?: FactOptions; +} + +/** + * @public + */ +export type Rule = { + conditions: TopLevelCondition; + name?: string; + priority?: number; +}; + +/** + * @public + */ +export interface TechInsightJsonRuleCheck extends TechInsightCheck { + rule: Rule; + dynamicFacts?: DynamicFact[]; +} + +/** + * @public + */ +export type CheckCondition = { + operator: string; + fact: string; + factValue: any; + factResult: any; + result: boolean; +}; + +/** + * @public + */ +export type ResponseTopLevelCondition = + | { all: CheckCondition[] } + | { any: CheckCondition[] }; + +/** + * @public + */ +export interface JsonRuleCheckResponse extends CheckResponse { + rule: { + conditions: ResponseTopLevelCondition & { + priority: number; + }; + }; +} + +/** + * @public + */ +export interface JsonRuleBooleanCheckResult extends BooleanCheckResult { + check: JsonRuleCheckResponse; +} diff --git a/plugins/tech-insights-backend/.eslintrc.js b/plugins/tech-insights-backend/.eslintrc.js new file mode 100644 index 0000000000..a126c438a4 --- /dev/null +++ b/plugins/tech-insights-backend/.eslintrc.js @@ -0,0 +1,11 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], + rules: { + 'jest/expect-expect': [ + 'error', + { + assertFunctionNames: ['expect', 'request.**.expect'], + }, + ], + }, +}; diff --git a/plugins/tech-insights-backend/CHANGELOG.md b/plugins/tech-insights-backend/CHANGELOG.md new file mode 100644 index 0000000000..17e0211ab0 --- /dev/null +++ b/plugins/tech-insights-backend/CHANGELOG.md @@ -0,0 +1,7 @@ +# @backstage/plugin-tech-insights-backend + +## 0.0.1 + +### Patch Changes + +- Initial implementation diff --git a/plugins/tech-insights-backend/README.md b/plugins/tech-insights-backend/README.md new file mode 100644 index 0000000000..4e5b5ad1b3 --- /dev/null +++ b/plugins/tech-insights-backend/README.md @@ -0,0 +1,211 @@ +# Tech Insights Backend + +This is the backend for the default Backstage Tech Insights feature. +This provides the API for the frontend tech insights, scorecards and fact visualization functionality, +as well as a framework to run fact retrievers and store fact values in to a data store. + +## Installation + +### Install the package + +```bash +# From your Backstage root directory +cd packages/backend +yarn add @backstage/plugin-tech-insights-backend +``` + +### Adding the plugin to your `packages/backend` + +You'll need to add the plugin to the router in your `backend` package. You can +do this by creating a file called `packages/backend/src/plugins/techInsights.ts`. An example content for `techInsights.ts` could be something like this. + +```ts +import { + createRouter, + DefaultTechInsightsBuilder, +} from '@backstage/plugin-tech-insights-backend'; +import { Router } from 'express'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin({ + logger, + config, + discovery, + database, +}: PluginEnvironment): Promise { + const builder = new DefaultTechInsightsBuilder({ + logger, + config, + database, + discovery, + factRetrievers: [], // Fact retrievers registrations you want tech insights to use + }); + + return await createRouter({ + ...(await builder.build()), + logger, + config, + }); +} +``` + +With the `techInsights.ts` router setup in place, add the router to +`packages/backend/src/index.ts`: + +```diff ++import techInsights from './plugins/techInsights'; + +async function main() { + ... + const createEnv = makeCreateEnv(config); + + const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); ++ const techInsightsEnv = useHotMemoize(module, () => createEnv('tech_insights')); + + const apiRouter = Router(); ++ apiRouter.use('/tech-insights', await techInsights(techInsightsEnv)); + ... + apiRouter.use(notFoundHandler()); + +``` + +### Adding fact retrievers + +At this point the Tech Insights backend is installed in your backend package, but +you will not have any fact retrievers present in your application. To have the implemented FactRetrieverEngine within this package to be able to retrieve and store fact data into the database, you need to add these. + +To create factRetrieverRegistration you need to implement `FactRetriever` interface defined in `@backstage/plugin-tech-insights-common` package. After you have implemented this interface you can wrap that into a registration object like follows: + +```ts +const myFactRetriever: FactRetriever = { + /** + * snip + */ +}; + +const myFactRetrieverRegistration = { + cadence: '1 * 3 * * ', // On the first minute of the third day of the month + factRetriever: myFactRetriever, +}; +``` + +Then you can modify the example `techInsights.ts` file shown above like this: + +```diff +const builder = new DefaultTechInsightsBuilder({ +logger, +config, +database, +discovery, +- factRetrievers: [], ++ factRetrievers: [myFactRetrieverRegistration], +}); +``` + +### Creating Fact Retrievers + +A Fact Retriever consist of three parts: + +1. `ref` - unique identifier of a fact retriever +2. `schema` - A versioned schema defining the shape of data a fact retriever returns +3. `handler` - An asynchronous function handling the logic of retrieving and returning facts for an entity + +An example implementation of a FactRetriever could for example be as follows: + +```ts +const myFactRetriever: FactRetriever = { + ref: 'documentation-number-factretriever', // unique ref, identifier of the fact retriever + schema: { + version: '0.1.1', // SemVer version number of this fact retriever schema. This should be incremented if the implementation changes + + // the actual schema + schema: { + // Name/identifier of an individual fact that this retriever returns + examplenumberfact: { + type: 'integer', // Type of the fact + description: 'A fact of a number', // Description of the fact + entityKinds: ['component'], // An array of entity kinds that this fact is applicable to + }, + }, + }, + handler: async ctx => { + // Handler function that retrieves the fact + const { discovery, config, logger } = ctx; + const catalogClient = new CatalogClient({ + discoveryApi: discovery, + }); + const entities = await catalogClient.getEntities(); // Retrieve all entities + /** + * snip: Do complex logic to retrieve facts from external system or calculate fact values + */ + + // Respond with an array of entity/fact values + return entities.items.map(it => { + return { + // Entity information that this fact relates to + entity: { + namespace: it.metadata.namespace, + kind: it.kind, + name: it.metadata.name, + }, + + // All facts that this retriever returns + facts: { + examplenumberfact: 2, // + }, + // (optional) timestamp to use as a Luxon DateTime object + }; + }); + }, +}; +``` + +### Adding a fact checker + +This module comes with a possibility to additionally add a fact checker and expose fact checking endpoints from the API. To be able to enable this feature you need to add a FactCheckerFactory implementation to be part of the `DefaultTechInsightsBuilder` constructor call. + +There is a default FactChecker implementation provided in module `@backstage/plugin-tech-insights-backend-module-jsonfc`. This implementation uses `json-rules-engine` as the underlying functionality to run checks. If you want to implement your own FactChecker, for example to be able to handle other than `boolean` result types, you can do so by implementing `FactCheckerFactory` and `FactChecker` interfaces from `@backstage/plugin-tech-insights-common` package. + +To add the default FactChecker into your Tech Insights you need to install the module into your backend application: + +```bash +# From your Backstage root directory +cd packages/backend +yarn add @backstage/plugin-tech-insights-backend-module-jsonfc +``` + +and modify the `techInsights.ts` file to contain a reference to the FactChecker implementation. + +```diff ++ import { JsonRulesEngineFactCheckerFactory } from '@backstage/plugin-tech-insights-backend-module-jsonfc'; + ++ const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ ++ checks: [], ++ logger, ++ }), + +const builder = new DefaultTechInsightsBuilder({ +logger, +config, +database, +discovery, +factRetrievers: [myFactRetrieverRegistration], ++ factCheckerFactory: myFactCheckerFactory +}); +``` + +To be able to run checks, you need to additionally add individual checks into your FactChecker implementation. For examples how to add these, you can check the documentation of the individual implementation of the FactChecker + +#### Modifying check persistence + +The default FactChecker implementation comes with an in-memory storage to store checks. You can inject an additional data store by adding an implementation of `TechInsightCheckRegistry` into the constructor options when creating a `JsonRulesEngineFactCheckerFactory`. That can be done as follows: + +```diff +const myTechInsightCheckRegistry: TechInsightCheckRegistry = // snip +const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ + checks: [], + logger, ++ checkRegistry: myTechInsightCheckRegistry +}), + +``` diff --git a/plugins/tech-insights-backend/api-report.md b/plugins/tech-insights-backend/api-report.md new file mode 100644 index 0000000000..fc1bd9217b --- /dev/null +++ b/plugins/tech-insights-backend/api-report.md @@ -0,0 +1,76 @@ +## API Report File for "@backstage/plugin-tech-insights-backend" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { CheckResult } from '@backstage/plugin-tech-insights-common'; +import { Config } from '@backstage/config'; +import express from 'express'; +import { FactChecker } from '@backstage/plugin-tech-insights-common'; +import { FactCheckerFactory } from '@backstage/plugin-tech-insights-common'; +import { FactRetrieverRegistration } from '@backstage/plugin-tech-insights-common'; +import { Logger as Logger_2 } from 'winston'; +import { PluginDatabaseManager } from '@backstage/backend-common'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { TechInsightCheck } from '@backstage/plugin-tech-insights-common'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-common'; + +// @public +export function createRouter< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +>(options: RouterOptions): Promise; + +// @public (undocumented) +export class DefaultTechInsightsBuilder< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + constructor(options: TechInsightsOptions); + build(): Promise>; +} + +// @public +export type PersistenceContext = { + techInsightsStore: TechInsightsStore; +}; + +// @public +export interface RouterOptions< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + config: Config; + factChecker?: FactChecker; + logger: Logger_2; + persistenceContext: PersistenceContext; +} + +// @public (undocumented) +export type TechInsightsContext< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> = { + factChecker?: FactChecker; + persistenceContext: PersistenceContext; +}; + +// @public (undocumented) +export interface TechInsightsOptions< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + // (undocumented) + config: Config; + // (undocumented) + database: PluginDatabaseManager; + // (undocumented) + discovery: PluginEndpointDiscovery; + factCheckerFactory?: FactCheckerFactory; + factRetrievers: FactRetrieverRegistration[]; + // (undocumented) + logger: Logger_2; +} + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js b/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js new file mode 100644 index 0000000000..4afe47061f --- /dev/null +++ b/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js @@ -0,0 +1,57 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.createTable('fact_schemas', table => { + table.comment( + 'The table for tech insight fact schemas. Containing a versioned data model definition for a collection of facts.', + ); + table.increments('id').primary(); + table + .text('ref') + .notNullable() + .comment('Identifier of the fact retriever plugin/package'); + table + .string('version') + .notNullable() + .comment('SemVer string defining the version of schema.'); + table + .text('schema') + .notNullable() + .comment( + 'Fact schema defining the values/types what this version of the fact would contain.', + ); + + table.index('ref', 'fact_schema_ref_idx'); + table.index(['ref', 'version'], 'fact_schema_ref_version_idx'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('fact_schemas', table => { + table.dropIndex([], 'fact_schema_ref_idx'); + table.dropIndex([], 'fact_schema_ref_version_idx'); + }); + await knex.schema.dropTable('fact_schemas'); +}; diff --git a/plugins/tech-insights-backend/migrations/202109061212_facts.js b/plugins/tech-insights-backend/migrations/202109061212_facts.js new file mode 100644 index 0000000000..f80883ef1d --- /dev/null +++ b/plugins/tech-insights-backend/migrations/202109061212_facts.js @@ -0,0 +1,73 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.createTable('facts', table => { + table.comment( + 'The table for tech insight fact collections. Contains facts for individual fact retriever namespace/ref.', + ); + table + .bigIncrements('index') + .notNullable() + .comment('An insert counter to ensure ordering'); + table + .text('ref') + .notNullable() + .comment('Unique identifier of the fact retriever plugin/package'); + table + .string('version') + .notNullable() + .comment( + 'SemVer string defining the version of schema this fact is based on.', + ); + table + .dateTime('timestamp') + .defaultTo(knex.fn.now()) + .notNullable() + .comment('The timestamp when this entry was created'); + table + .text('entity') + .notNullable() + .comment('Identifier of the entity these facts relate to'); + table + .text('facts') + .notNullable() + .comment( + 'Values of the fact collection stored as key-value pairs in JSON format.', + ); + + table.index('index', 'fact_index_idx'); + table.index('ref', 'fact_ref_idx'); + table.index(['ref', 'entity'], 'fact_ref_entity_idx'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('facts', table => { + table.dropIndex([], 'facts_index_idx'); + table.dropIndex([], 'fact_ref_idx'); + table.dropIndex([], 'fact_ref_entity_idx'); + }); + await knex.schema.dropTable('facts'); +}; diff --git a/plugins/tech-insights-backend/package.json b/plugins/tech-insights-backend/package.json new file mode 100644 index 0000000000..fba5daae45 --- /dev/null +++ b/plugins/tech-insights-backend/package.json @@ -0,0 +1,64 @@ +{ + "name": "@backstage/plugin-tech-insights-backend", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/tech-insights-backend" + }, + "keywords": [ + "backstage", + "tech-insights", + "reporting" + ], + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/backend-common": "^0.9.0", + "@backstage/catalog-client": "^0.3.16", + "@backstage/catalog-model": "^0.9.0", + "@backstage/config": "^0.1.8", + "@backstage/errors": "^0.1.1", + "@backstage/plugin-tech-insights-common": "^0.1.0", + "@types/express": "^4.17.6", + "cross-fetch": "^3.0.6", + "express": "^4.17.1", + "express-promise-router": "^4.1.0", + "knex": "^0.95.1", + "lodash": "^4.17.21", + "luxon": "^2.0.2", + "node-cron": "^3.0.0", + "semver": "^7.3.5", + "uuid": "^8.3.2", + "winston": "^3.2.1", + "yn": "^4.0.0" + }, + "devDependencies": { + "@backstage/cli": "^0.7.1", + "@types/supertest": "^2.0.8", + "@types/node-cron": "^3.0.0", + "@types/semver": "^7.3.8", + "supertest": "^6.1.3" + }, + "files": [ + "dist", + "migrations/**/*.{js,d.ts}" + ] +} diff --git a/plugins/tech-insights-backend/src/index.ts b/plugins/tech-insights-backend/src/index.ts new file mode 100644 index 0000000000..eb54e8cfed --- /dev/null +++ b/plugins/tech-insights-backend/src/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './service/router'; +export type { RouterOptions } from './service/router'; + +export { DefaultTechInsightsBuilder } from './service/DefaultTechInsightsBuilder'; +export type { + TechInsightsOptions, + TechInsightsContext, +} from './service/DefaultTechInsightsBuilder'; + +export type { PersistenceContext } from './service/persistence/DatabaseManager'; diff --git a/plugins/tech-insights-backend/src/service/DefaultTechInsightsBuilder.ts b/plugins/tech-insights-backend/src/service/DefaultTechInsightsBuilder.ts new file mode 100644 index 0000000000..6c5b946c4c --- /dev/null +++ b/plugins/tech-insights-backend/src/service/DefaultTechInsightsBuilder.ts @@ -0,0 +1,149 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { FactRetrieverEngine } from './fact/FactRetrieverEngine'; +import { Logger } from 'winston'; +import { FactRetrieverRegistry } from './fact/FactRetrieverRegistry'; +import { Config } from '@backstage/config'; +import { + PluginDatabaseManager, + PluginEndpointDiscovery, +} from '@backstage/backend-common'; +import { + CheckResult, + FactChecker, + FactCheckerFactory, + FactRetrieverRegistration, + TechInsightCheck, +} from '@backstage/plugin-tech-insights-common'; +import { + DatabaseManager, + PersistenceContext, +} from './persistence/DatabaseManager'; + +/** + * @public + * @typeParam CheckType - Type of the check for the fact checker this builder returns + * @typeParam CheckResultType - Type of the check result for the fact checker this builder returns + * + * Configuration options to initialize TechInsightsBuilder. Generic types params are needed if FactCheckerFactory + * is included for FactChecker creation. + */ +export interface TechInsightsOptions< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + /** + * A collection of FactRetrieverRegistrations. + * Used to register FactRetrievers and their schemas and schedule an execution loop for them. + */ + factRetrievers: FactRetrieverRegistration[]; + + /** + * Optional factory exposing a `construct` method to initialize a FactChecker implementation + */ + factCheckerFactory?: FactCheckerFactory; + + logger: Logger; + config: Config; + discovery: PluginEndpointDiscovery; + database: PluginDatabaseManager; +} + +/** + * @public + * @typeParam CheckType - Type of the check for the fact checker this builder returns + * @typeParam CheckResultType - Type of the check result for the fact checker this builder returns + * + * A container for exported implementations related to TechInsights. + * FactChecker is present if an optional FactCheckerFactory is included in the build stage. + */ +export type TechInsightsContext< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> = { + factChecker?: FactChecker; + persistenceContext: PersistenceContext; +}; + +/** + * @public + * @typeParam CheckType - Type of the check for the fact checker this builder returns + * @typeParam CheckResultType - Type of the check result for the fact checker this builder returns + * + * Default implementation of TechInsightsBuilder. + */ +export class DefaultTechInsightsBuilder< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + private readonly options: TechInsightsOptions; + + constructor(options: TechInsightsOptions) { + this.options = options; + } + + /** + * Constructs needed persistence context, fact retriever engine + * and optionally fact checker implementations to be used in the tech insights module. + * + * @returns TechInsightsContext with persistence implementations and optionally an implementation of a FactChecker + */ + async build(): Promise> { + const { + factRetrievers, + factCheckerFactory, + config, + discovery, + database, + logger, + } = this.options; + + const factRetrieverRegistry = new FactRetrieverRegistry(factRetrievers); + + const persistenceContext = + await DatabaseManager.initializePersistenceContext( + await database.getClient(), + { logger }, + ); + + const factRetrieverEngine = await FactRetrieverEngine.fromConfig({ + repository: persistenceContext.techInsightsStore, + factRetrieverRegistry, + factRetrieverContext: { + config, + discovery, + logger, + }, + }); + + factRetrieverEngine.schedule(); + + if (factCheckerFactory) { + const factChecker = factCheckerFactory.construct( + persistenceContext.techInsightsStore, + ); + return { + persistenceContext, + factChecker, + }; + } + + return { + persistenceContext, + }; + } +} diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts new file mode 100644 index 0000000000..6b9b58e3b8 --- /dev/null +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts @@ -0,0 +1,152 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + FactRetriever, + FactRetrieverRegistration, + FactSchema, + TechInsightFact, + TechInsightsStore, +} from '@backstage/plugin-tech-insights-common'; +import { FactRetrieverRegistry } from './FactRetrieverRegistry'; +import { FactRetrieverEngine } from './FactRetrieverEngine'; +import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { schedule } from 'node-cron'; + +jest.mock('node-cron', () => { + const original = jest.requireActual('node-cron'); + return { + ...original, + schedule: jest.fn(), + }; +}); + +const testFactRetriever: FactRetriever = { + ref: 'test-factretriever', + schema: { + version: '0.0.1', + schema: { + testnumberfact: { + type: 'integer', + description: '', + entityKinds: ['component'], + }, + }, + }, + handler: async () => { + return [ + { + ref: 'test-factretriever', + entity: { + namespace: 'a', + kind: 'a', + name: 'a', + }, + facts: { + testnumberfact: 1, + }, + }, + ]; + }, +}; +const cadence = '1 * * * *'; +describe('FactRetrieverEngine', () => { + let engine: FactRetrieverEngine; + let factSchemaAssertionCallback: (ref: string, schema: FactSchema) => void; + let factInsertionAssertionCallback: (facts: TechInsightFact[]) => void; + + const mockRepository: TechInsightsStore = { + insertFacts: (facts: TechInsightFact[]) => { + factInsertionAssertionCallback(facts); + return Promise.resolve(); + }, + insertFactSchema: (ref: string, schema: FactSchema) => { + factSchemaAssertionCallback(ref, schema); + return Promise.resolve(); + }, + } as unknown as TechInsightsStore; + + const mockFactRetrieverRegistry: FactRetrieverRegistry = { + listRetrievers(): FactRetriever[] { + return [testFactRetriever]; + }, + listRegistrations(): FactRetrieverRegistration[] { + return [{ factRetriever: testFactRetriever, cadence }]; + }, + } as unknown as FactRetrieverRegistry; + + const defaultEngineConfig = { + factRetrieverContext: { + logger: getVoidLogger(), + config: ConfigReader.fromConfigs([]), + discovery: { + getBaseUrl: (_: string) => Promise.resolve('http://mock.url'), + getExternalBaseUrl: (_: string) => Promise.resolve('http://mock.url'), + }, + }, + factRetrieverRegistry: mockFactRetrieverRegistry, + repository: mockRepository, + }; + + it('Should update fact retriever schemas on initialization', async () => { + factSchemaAssertionCallback = (ref, schema) => { + expect(ref).toEqual('test-factretriever'); + expect(schema).toEqual({ + version: '0.0.1', + schema: { + testnumberfact: { + type: 'integer', + description: '', + entityKinds: ['component'], + }, + }, + }); + }; + + engine = await FactRetrieverEngine.fromConfig(defaultEngineConfig); + }); + it('Should insert facts when scheduled step is run', async () => { + (schedule as jest.Mock).mockImplementation( + (cronCadence: string, retrieverAction: Function) => { + return { + cadence: cronCadence, + triggerScheduledJobNow: retrieverAction, + }; + }, + ); + + factSchemaAssertionCallback = () => {}; + factInsertionAssertionCallback = facts => { + expect(facts).toHaveLength(1); + expect(facts[0]).toEqual({ + ref: 'test-factretriever', + entity: { + namespace: 'a', + kind: 'a', + name: 'a', + }, + facts: { + testnumberfact: 1, + }, + }); + }; + engine = await FactRetrieverEngine.fromConfig(defaultEngineConfig); + engine.schedule(); + const job: any = engine.getJob('test-factretriever'); + job.triggerScheduledJobNow(); + expect(job.cadence!!).toEqual(cadence); + }); +}); diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts new file mode 100644 index 0000000000..01e3c12370 --- /dev/null +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts @@ -0,0 +1,135 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + FactRetriever, + FactRetrieverContext, + TechInsightsStore, +} from '@backstage/plugin-tech-insights-common'; +import { FactRetrieverRegistry } from './FactRetrieverRegistry'; +import { schedule, validate, ScheduledTask } from 'node-cron'; +import { Logger } from 'winston'; + +function randomDailyCron() { + const rand = (min: number, max: number) => + Math.floor(Math.random() * (max - min + 1) + min); + return `${rand(0, 59)} ${rand(0, 23)} * * *`; +} + +function duration(startTimestamp: [number, number]): string { + const delta = process.hrtime(startTimestamp); + const seconds = delta[0] + delta[1] / 1e9; + return `${seconds.toFixed(1)}s`; +} + +export class FactRetrieverEngine { + private scheduledJobs = new Map(); + + constructor( + private readonly repository: TechInsightsStore, + private readonly factRetrieverRegistry: FactRetrieverRegistry, + private readonly factRetrieverContext: FactRetrieverContext, + private readonly logger: Logger, + private readonly defaultCadence?: string, + ) {} + + static async fromConfig({ + repository, + factRetrieverRegistry, + factRetrieverContext, + defaultCadence, + }: { + repository: TechInsightsStore; + factRetrieverRegistry: FactRetrieverRegistry; + factRetrieverContext: FactRetrieverContext; + defaultCadence?: string; + }) { + await Promise.all( + factRetrieverRegistry + .listRetrievers() + .map(it => repository.insertFactSchema(it.ref, it.schema)), + ); + + return new FactRetrieverEngine( + repository, + factRetrieverRegistry, + factRetrieverContext, + factRetrieverContext.logger, + defaultCadence, + ); + } + + schedule() { + const registrations = this.factRetrieverRegistry.listRegistrations(); + const newRegs: string[] = []; + registrations.forEach(registration => { + const { factRetriever, cadence } = registration; + if (!this.scheduledJobs.has(factRetriever.ref)) { + const cronExpression = + cadence || this.defaultCadence || randomDailyCron(); + if (!validate(cronExpression)) { + this.logger.warn( + `Validation failed for cron expression ${cronExpression} when trying to schedule fact retriever ${factRetriever.ref}`, + ); + return; + } + const job = schedule( + cronExpression, + this.createFactRetrieverHandler(factRetriever), + ); + this.scheduledJobs.set(factRetriever.ref, job); + newRegs.push(factRetriever.ref); + } + }); + this.logger.info( + `Scheduled ${newRegs.length} fact retrievers to Fact Retriever Engine.`, + ); + } + + getJob(ref: string) { + return this.scheduledJobs.get(ref); + } + + private createFactRetrieverHandler(factRetriever: FactRetriever) { + return async () => { + const startTimestamp = process.hrtime(); + this.logger.info( + `Retrieving facts for fact retriever ${factRetriever.ref}`, + ); + const facts = await factRetriever.handler(this.factRetrieverContext); + if (this.logger.isDebugEnabled()) { + this.logger.debug( + `Retrieved ${facts.length} facts for fact retriever ${ + factRetriever.ref + } in ${duration(startTimestamp)}`, + ); + } + + try { + await this.repository.insertFacts(factRetriever.ref, facts); + this.logger.info( + `Stored ${facts.length} facts for fact retriever ${ + factRetriever.ref + } in ${duration(startTimestamp)}`, + ); + } catch (e) { + this.logger.warn( + `Failed to insert facts for fact retriever ${factRetriever.ref}`, + e, + ); + } + }; + } +} diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts new file mode 100644 index 0000000000..c76bb6039d --- /dev/null +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + FactRetriever, + FactRetrieverRegistration, + FactSchema, +} from '@backstage/plugin-tech-insights-common'; +import { ConflictError, NotFoundError } from '@backstage/errors'; + +export class FactRetrieverRegistry { + private readonly retrievers = new Map(); + + constructor(retrievers: FactRetrieverRegistration[]) { + retrievers.forEach(it => { + this.register(it); + }); + } + + register(registration: FactRetrieverRegistration) { + if (this.retrievers.has(registration.factRetriever.ref)) { + throw new ConflictError( + `Tech insight fact retriever with reference '${registration.factRetriever.ref}' has already been registered`, + ); + } + this.retrievers.set(registration.factRetriever.ref, registration); + } + + get(retrieverReference: string): FactRetriever { + const registration = this.retrievers.get(retrieverReference); + if (!registration) { + throw new NotFoundError( + `Tech insight fact retriever with reference '${retrieverReference}' is not registered.`, + ); + } + return registration.factRetriever; + } + + listRetrievers(): FactRetriever[] { + return [...this.retrievers.values()].map(it => it.factRetriever); + } + + listRegistrations(): FactRetrieverRegistration[] { + return [...this.retrievers.values()]; + } + + getSchemas(): FactSchema[] { + return this.listRetrievers().map(it => it.schema); + } +} diff --git a/plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts b/plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts new file mode 100644 index 0000000000..75fb313f80 --- /dev/null +++ b/plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts @@ -0,0 +1,99 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common'; +import knexFactory, { Knex } from 'knex'; +import { Logger } from 'winston'; +import { v4 as uuidv4 } from 'uuid'; +import { TechInsightsDatabase } from './TechInsightsDatabase'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-common'; + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-tech-insights-backend', + 'migrations', +); + +/** + * A Container for persistence related components in TechInsights + * + * @public + */ +export type PersistenceContext = { + techInsightsStore: TechInsightsStore; +}; + +export type CreateDatabaseOptions = { + logger: Logger; +}; + +const defaultOptions: CreateDatabaseOptions = { + logger: getVoidLogger(), +}; + +/** + * A factory class to construct persistence context for both running implmentation and test cases. + * + * @public + */ +export class DatabaseManager { + public static async initializePersistenceContext( + knex: Knex, + options: CreateDatabaseOptions = defaultOptions, + ): Promise { + await knex.migrate.latest({ + directory: migrationsDir, + }); + return { + techInsightsStore: new TechInsightsDatabase(knex, options.logger), + }; + } + + public static async createTestDatabase( + knex: Knex, + ): Promise { + const knexInstance = knex ?? (await this.createTestDatabaseConnection()); + return await this.initializePersistenceContext(knexInstance); + } + + public static async createTestDatabaseConnection(): Promise { + const config: Knex.Config = { + client: 'sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }; + + let knexInstance = knexFactory(config); + if (typeof config.connection !== 'string') { + const tempDbName = `d${uuidv4().replace(/-/g, '')}`; + await knexInstance.raw(`CREATE DATABASE ${tempDbName};`); + knexInstance = knexFactory({ + ...config, + connection: { + ...config.connection, + database: tempDbName, + }, + }); + } + + knexInstance.client.pool.on( + 'createSuccess', + (_eventId: any, resource: any) => { + resource.run('PRAGMA foreign_keys = ON', () => {}); + }, + ); + + return knexInstance; + } +} diff --git a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts new file mode 100644 index 0000000000..ab54cf819d --- /dev/null +++ b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts @@ -0,0 +1,218 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { DatabaseManager } from './DatabaseManager'; +import { DateTime, Duration } from 'luxon'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-common'; +import { Knex } from 'knex'; + +const factSchemas = [ + { + ref: 'test-schema', + version: '0.0.1-test', + schema: JSON.stringify({ + testNumberFact: { + type: 'integer', + description: 'Test fact with a number type', + entityKinds: ['component'], + }, + }), + }, +]; +const additionalFactSchemas = [ + { + ref: 'test-schema', + version: '1.2.1-test', + schema: JSON.stringify({ + testNumberFact: { + type: 'integer', + description: 'Test fact with a number type', + entityKinds: ['component'], + }, + testStringFact: { + type: 'string', + description: 'Test fact with a string type', + entityKinds: ['service'], + }, + }), + }, + { + ref: 'test-schema', + version: '1.1.1-test', + schema: JSON.stringify({ + testStringFact: { + type: 'string', + description: 'Test fact with a string type', + entityKinds: ['service'], + }, + }), + }, +]; + +const now = DateTime.now().toISO(); +const shortlyInTheFuture = DateTime.now() + .plus(Duration.fromMillis(555)) + .toISO(); +const farInTheFuture = DateTime.now() + .plus(Duration.fromMillis(555666777)) + .toISO(); + +const facts = [ + { + timestamp: now, + ref: 'test-fact', + version: '0.0.1-test', + entity: 'a/a/a', + facts: JSON.stringify({ + testNumberFact: 1, + }), + }, + { + timestamp: shortlyInTheFuture, + ref: 'test-fact', + version: '0.0.1-test', + entity: 'a/a/a', + facts: JSON.stringify({ + testNumberFact: 2, + }), + }, +]; + +const additionalFacts = [ + { + timestamp: farInTheFuture, + ref: 'test-fact', + version: '0.0.1-test', + entity: 'a/a/a', + facts: JSON.stringify({ + testNumberFact: 3, + }), + }, +]; + +describe('Tech Insights database', () => { + let store: TechInsightsStore; + let testDbClient: Knex; + beforeAll(async () => { + testDbClient = await DatabaseManager.createTestDatabaseConnection(); + store = (await DatabaseManager.createTestDatabase(testDbClient)) + .techInsightsStore; + await testDbClient.batchInsert('fact_schemas', factSchemas); + await testDbClient.batchInsert('facts', facts); + }); + + const baseAssertionFact = { + ref: 'test-fact', + entity: { namespace: 'a', kind: 'a', name: 'a' }, + timestamp: DateTime.fromISO(shortlyInTheFuture), + version: '0.0.1-test', + facts: { testNumberFact: 2 }, + }; + + it('should be able to return latest schema', async () => { + const schemas = await store.getLatestSchemas(); + expect(schemas[0]).toMatchObject({ + ref: 'test-schema', + version: '0.0.1-test', + schema: { + testNumberFact: { + type: 'integer', + description: 'Test fact with a number type', + entityKinds: ['component'], + }, + }, + }); + }); + + it('should return last schema based on semver', async () => { + await testDbClient.batchInsert('fact_schemas', additionalFactSchemas); + + const schemas = await store.getLatestSchemas(); + expect(schemas[0]).toMatchObject({ + ref: 'test-schema', + version: '1.2.1-test', + schema: { + testNumberFact: { + type: 'integer', + description: 'Test fact with a number type', + entityKinds: ['component'], + }, + testStringFact: { + type: 'string', + description: 'Test fact with a string type', + entityKinds: ['service'], + }, + }, + }); + }); + + it('should return latest facts only for the correct ref', async () => { + const returnedFact = await store.getLatestFactsForRefs( + ['test-fact'], + 'a/a/a', + ); + expect(returnedFact['test-fact']).toMatchObject(baseAssertionFact); + }); + + it('should return latest facts for multiple refs', async () => { + await testDbClient.batchInsert( + 'facts', + additionalFacts.map(fact => ({ + ...fact, + ref: 'second-test-fact', + timestamp: farInTheFuture, + })), + ); + const returnedFacts = await store.getLatestFactsForRefs( + ['test-fact', 'second-test-fact'], + 'a/a/a', + ); + + expect(returnedFacts['test-fact']).toMatchObject({ + ...baseAssertionFact, + }); + expect(returnedFacts['second-test-fact']).toMatchObject({ + ...baseAssertionFact, + ref: 'second-test-fact', + timestamp: DateTime.fromISO(farInTheFuture), + facts: { testNumberFact: 3 }, + }); + }); + + it('should return facts correctly between time range', async () => { + await testDbClient.batchInsert('facts', additionalFacts); + const returnedFacts = await store.getFactsBetweenTimestampsForRefs( + ['test-fact'], + 'a/a/a', + DateTime.fromISO(now), + DateTime.fromISO(shortlyInTheFuture).plus(Duration.fromMillis(10)), + ); + expect(returnedFacts['test-fact']).toHaveLength(2); + + expect(returnedFacts['test-fact'][0]).toMatchObject({ + ...baseAssertionFact, + timestamp: DateTime.fromISO(now), + facts: { testNumberFact: 1 }, + }); + expect(returnedFacts['test-fact'][1]).toMatchObject({ + ...baseAssertionFact, + }); + expect(returnedFacts['test-fact']).not.toContainEqual({ + ...baseAssertionFact, + timestamp: DateTime.fromISO(farInTheFuture), + facts: { testNumberFact: 3 }, + }); + }); +}); diff --git a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts new file mode 100644 index 0000000000..1bdf18249f --- /dev/null +++ b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts @@ -0,0 +1,188 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Knex } from 'knex'; +import { + FactSchema, + TechInsightFact, + FlatTechInsightFact, + TechInsightsStore, +} from '@backstage/plugin-tech-insights-common'; +import { rsort } from 'semver'; +import { groupBy } from 'lodash'; +import { DateTime } from 'luxon'; +import { Logger } from 'winston'; + +export type RawDbFactRow = { + ref: string; + version: string; + timestamp: Date | string; + entity: string; + facts: string; +}; + +type RawDbFactSchemaRow = { + id: number; + ref: string; + version: string; + schema: string; +}; + +export class TechInsightsDatabase implements TechInsightsStore { + private readonly CHUNK_SIZE = 50; + + constructor(private readonly db: Knex, private readonly logger: Logger) {} + + async getLatestSchemas(refs?: string[]): Promise { + const queryBuilder = this.db('fact_schemas'); + if (refs) { + queryBuilder.whereIn('ref', refs); + } + const existingSchemas = await queryBuilder.orderBy('id', 'desc').select(); + + const groupedSchemas = groupBy(existingSchemas, 'ref'); + return Object.values(groupedSchemas) + .map(schemas => { + const sorted = rsort(schemas.map(it => it.version)); + return schemas.find(it => it.version === sorted[0])!!; + }) + .map((it: RawDbFactSchemaRow) => ({ + ...it, + schema: JSON.parse(it.schema), + })); + } + + async insertFactSchema(ref: string, schema: FactSchema) { + const existingSchemas = await this.db('fact_schemas') + .where({ ref }) + .select(); + const exists = existingSchemas.some( + it => it.ref === ref && it.version === schema.version, + ); + + if (!exists) { + await this.db('fact_schemas').insert({ + ref, + version: schema.version, + schema: JSON.stringify(schema.schema), + }); + } + } + + async insertFacts(ref: string, facts: TechInsightFact[]): Promise { + if (facts.length === 0) return; + const currentSchema = await this.getLatestSchema(ref); + const factRows = facts.map(it => { + const { namespace, name, kind } = it.entity; + return { + ref: ref, + version: currentSchema.version, + entity: `${namespace}/${kind}/${name}`.toLocaleLowerCase('en-US'), + facts: JSON.stringify(it.facts), + ...(it.timestamp && { timestamp: it.timestamp.toJSDate() }), + }; + }); + await this.db.transaction(async tx => { + await tx.batchInsert('facts', factRows, this.CHUNK_SIZE); + }); + } + + async getLatestFactsForRefs( + refs: string[], + entityTriplet: string, + ): Promise<{ [p: string]: FlatTechInsightFact }> { + const results = await this.db('facts') + .where({ entity: entityTriplet }) + .and.whereIn('ref', refs) + .join( + this.db('facts') + .max('timestamp') + .column('ref as subRef') + .groupBy('ref') + .as('subQ'), + 'facts.ref', + 'subQ.subRef', + ); + return this.dbFactRowsToTechInsightFacts(results); + } + + async getFactsBetweenTimestampsForRefs( + refs: string[], + entityTriplet: string, + startDateTime: DateTime, + endDateTime: DateTime, + ): Promise<{ + [p: string]: FlatTechInsightFact[]; + }> { + const results = await this.db('facts') + .where({ entity: entityTriplet }) + .and.whereIn('ref', refs) + .and.whereBetween('timestamp', [ + startDateTime.toISO(), + endDateTime.toISO(), + ]); + + return groupBy( + results.map(it => { + const [namespace, kind, name] = it.entity.split('/'); + const timestamp = + typeof it.timestamp === 'string' + ? DateTime.fromISO(it.timestamp) + : DateTime.fromJSDate(it.timestamp); + return { + ref: it.ref, + entity: { namespace, kind, name }, + timestamp, + version: it.version, + facts: JSON.parse(it.facts), + }; + }), + 'ref', + ); + } + + private async getLatestSchema(ref: string): Promise { + const existingSchemas = await this.db('fact_schemas') + .where({ ref }) + .orderBy('id', 'desc') + .select(); + if (existingSchemas.length < 1) { + this.logger.warn(`No schema found for ${ref}. `); + throw new Error(`No schema found for ${ref}. `); + } + const sorted = rsort(existingSchemas.map(it => it.version)); + return existingSchemas.find(it => it.version === sorted[0])!!; + } + + private dbFactRowsToTechInsightFacts(rows: RawDbFactRow[]) { + return rows.reduce((acc, it) => { + const [namespace, kind, name] = it.entity.split('/'); + const timestamp = + typeof it.timestamp === 'string' + ? DateTime.fromISO(it.timestamp) + : DateTime.fromJSDate(it.timestamp); + return { + ...acc, + [it.ref]: { + ref: it.ref, + entity: { namespace, kind, name }, + timestamp, + version: it.version, + facts: JSON.parse(it.facts), + }, + }; + }, {}); + } +} diff --git a/plugins/tech-insights-backend/src/service/router.test.ts b/plugins/tech-insights-backend/src/service/router.test.ts new file mode 100644 index 0000000000..1433722e6c --- /dev/null +++ b/plugins/tech-insights-backend/src/service/router.test.ts @@ -0,0 +1,125 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { DefaultTechInsightsBuilder } from './DefaultTechInsightsBuilder'; +import { createRouter } from './router'; +import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import request from 'supertest'; +import express from 'express'; +import { PersistenceContext } from './persistence/DatabaseManager'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-common'; +import { DateTime } from 'luxon'; +import { Knex } from 'knex'; + +describe('Tech Insights router tests', () => { + let app: express.Express; + + const latestFactsForRefsMock = jest.fn(); + const factsBetweenTimestampsForRefsMock = jest.fn(); + const latestSchemasMock = jest.fn(); + + const mockPersistenceContext: PersistenceContext = { + techInsightsStore: { + getLatestFactsForRefs: latestFactsForRefsMock, + getFactsBetweenTimestampsForRefs: factsBetweenTimestampsForRefsMock, + getLatestSchemas: latestSchemasMock, + } as unknown as TechInsightsStore, + }; + + afterEach(() => { + jest.resetAllMocks(); + }); + + beforeAll(async () => { + const techInsightsContext = await new DefaultTechInsightsBuilder({ + database: { + getClient: () => { + return Promise.resolve({ + migrate: { + latest: () => {}, + }, + }) as unknown as Promise; + }, + }, + logger: getVoidLogger(), + factRetrievers: [], + config: ConfigReader.fromConfigs([]), + discovery: { + getBaseUrl: (_: string) => Promise.resolve('http://mock.url'), + getExternalBaseUrl: (_: string) => Promise.resolve('http://mock.url'), + }, + }).build(); + + const router = await createRouter({ + logger: getVoidLogger(), + config: ConfigReader.fromConfigs([]), + ...techInsightsContext, + persistenceContext: mockPersistenceContext, + }); + + app = express().use(router); + }); + + it('should be able to retrieve latest schemas', async () => { + await request(app).get('/fact-schemas').expect(200); + expect(latestSchemasMock).toHaveBeenCalled(); + }); + + it('should not contain check endpoints when checker not present', async () => { + await request(app).get('/checks').expect(404); + await request(app).get('/checks/a/a/a').expect(404); + }); + + it('should parse be able to parse ref request params for fact retrieval', async () => { + await request(app) + .get('/facts/latest/a/a/a') + .query({ refs: ['firstref', 'secondref'] }) + .expect(200); + expect(latestFactsForRefsMock).toHaveBeenCalledWith( + ['firstref', 'secondref'], + 'a/a/a', + ); + }); + + it('should parse be able to parse datetime request params for fact retrieval', async () => { + await request(app) + .get('/facts/range/a/a/a') + .query({ + refs: ['firstref', 'secondref'], + startDatetime: '2021-12-12T12:12:12', + endDatetime: '2022-11-11T11:11:11', + }) + .expect(200); + expect(factsBetweenTimestampsForRefsMock).toHaveBeenCalledWith( + ['firstref', 'secondref'], + 'a/a/a', + DateTime.fromISO('2021-12-12T12:12:12.000+00:00'), + DateTime.fromISO('2022-11-11T11:11:11.000+00:00'), + ); + }); + + it('should respond gracefully on parsing errors', async () => { + await request(app) + .get('/facts/range/a/a/a') + .query({ + refs: ['firstref', 'secondref'], + startDatetime: '2021-12-1222T12:12:12', + endDatetime: '2022-1122-11T11:11:11', + }) + .expect(422); + expect(latestFactsForRefsMock).toHaveBeenCalledTimes(0); + }); +}); diff --git a/plugins/tech-insights-backend/src/service/router.ts b/plugins/tech-insights-backend/src/service/router.ts new file mode 100644 index 0000000000..3649f522dd --- /dev/null +++ b/plugins/tech-insights-backend/src/service/router.ts @@ -0,0 +1,136 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import express from 'express'; +import Router from 'express-promise-router'; +import { Config } from '@backstage/config'; +import { + FactChecker, + TechInsightCheck, + CheckResult, +} from '@backstage/plugin-tech-insights-common'; +import { Logger } from 'winston'; +import { DateTime } from 'luxon'; +import { PersistenceContext } from './persistence/DatabaseManager'; + +/** + * @public + * + * RouterOptions to construct TechInsights endpoints + * @typeParam CheckType - Type of the check for the fact checker this builder returns + * @typeParam CheckResultType - Type of the check result for the fact checker this builder returns + */ +export interface RouterOptions< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + /** + * Optional FactChecker implementation. If omitted, endpoints are not constructed + */ + factChecker?: FactChecker; + + /** + * TechInsights PersistenceContext. Should contain an implementation of TechInsightsStore + */ + persistenceContext: PersistenceContext; + + /** + * Backstage config object + */ + config: Config; + + /** + * Implementation of Winston logger + */ + logger: Logger; +} + +/** + * @public + * + * Constructs a tech-insights router. + * + * Exposes endpoints to handle facts + * Exposes optional endpoints to handle checks if a FactChecker implementation is passed in + * + * @param options - RouterOptions object + */ +export async function createRouter< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +>(options: RouterOptions): Promise { + const router = Router(); + router.use(express.json()); + const { persistenceContext, factChecker, logger } = options; + const { techInsightsStore } = persistenceContext; + + if (factChecker) { + logger.info('Fact checker configured. Enabling fact checking endpoints.'); + router.get('/checks', async (_req, res) => { + return res.send(await factChecker.getChecks()); + }); + + router.get('/checks/:namespace/:kind/:name', async (req, res) => { + const { namespace, kind, name } = req.params; + const checks = req.query.checks as string[]; + const entityTriplet = `${namespace.toLowerCase()}/${kind.toLowerCase()}/${name.toLowerCase()}`; + try { + const checkResult = await factChecker.runChecks(entityTriplet, checks); + return res.send(checkResult); + } catch (e) { + return res.status(500).json({ message: e.message }).send(); + } + }); + } else { + logger.info( + 'Starting tech insights module without fact checking endpoints.', + ); + } + + router.get('/fact-schemas', async (req, res) => { + const refs = req.query.refs as string[]; + return res.send(await techInsightsStore.getLatestSchemas(refs)); + }); + + router.get('/facts/latest/:namespace/:kind/:name', async (req, res) => { + const { namespace, kind, name } = req.params; + const refs = req.query.refs as string[]; + const entityTriplet = `${namespace.toLowerCase()}/${kind.toLowerCase()}/${name.toLowerCase()}`; + return res.send( + await techInsightsStore.getLatestFactsForRefs(refs, entityTriplet), + ); + }); + + router.get('/facts/range/:namespace/:kind/:name', async (req, res) => { + const { namespace, kind, name } = req.params; + const refs = req.query.refs as string[]; + const startDatetime = DateTime.fromISO(req.query.startDatetime as string); + const endDatetime = DateTime.fromISO(req.query.endDatetime as string); + if (!startDatetime.isValid || !endDatetime.isValid) { + return res.status(422).send('Failed to parse datetime from request'); + } + const entityTriplet = `${namespace.toLowerCase()}/${kind.toLowerCase()}/${name.toLowerCase()}`; + return res.send( + await techInsightsStore.getFactsBetweenTimestampsForRefs( + refs, + entityTriplet, + startDatetime, + endDatetime, + ), + ); + }); + return router; +} diff --git a/plugins/tech-insights-backend/src/setupTests.ts b/plugins/tech-insights-backend/src/setupTests.ts new file mode 100644 index 0000000000..a330613afb --- /dev/null +++ b/plugins/tech-insights-backend/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/plugins/tech-insights-common/.eslintrc.js b/plugins/tech-insights-common/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/tech-insights-common/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/tech-insights-common/CHANGELOG.md b/plugins/tech-insights-common/CHANGELOG.md new file mode 100644 index 0000000000..17e0211ab0 --- /dev/null +++ b/plugins/tech-insights-common/CHANGELOG.md @@ -0,0 +1,7 @@ +# @backstage/plugin-tech-insights-backend + +## 0.0.1 + +### Patch Changes + +- Initial implementation diff --git a/plugins/tech-insights-common/README.md b/plugins/tech-insights-common/README.md new file mode 100644 index 0000000000..651c0e969d --- /dev/null +++ b/plugins/tech-insights-common/README.md @@ -0,0 +1,3 @@ +# Tech Insights Common + +Common types and functionalities for tech insights, to be shared between tech-insights-backend and individual implementations of FactRetrievers, FactCheckers and their persistence options. diff --git a/plugins/tech-insights-common/api-report.md b/plugins/tech-insights-common/api-report.md new file mode 100644 index 0000000000..6d0d40239b --- /dev/null +++ b/plugins/tech-insights-common/api-report.md @@ -0,0 +1,167 @@ +## API Report File for "@backstage/plugin-tech-insights-common" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { Config } from '@backstage/config'; +import { DateTime } from 'luxon'; +import { Logger as Logger_2 } from 'winston'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; + +// @public +export interface BooleanCheckResult extends CheckResult { + // (undocumented) + result: boolean; +} + +// @public +export interface CheckResponse { + description: string; + factRefs: string[]; + id: string; + metadata?: Record; + name: string; +} + +// @public +export type CheckResult = { + facts: FactResponse; + check: CheckResponse; +}; + +// @public +export interface FactChecker< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + addCheck(check: CheckType): Promise; + getChecks(): Promise; + runChecks(entity: string, checks: string[]): Promise; + validate(check: CheckType): Promise; +} + +// @public +export interface FactCheckerFactory< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + // (undocumented) + construct( + repository: TechInsightsStore, + ): FactChecker; +} + +// @public +export type FactResponse = { + [key: string]: { + ref: string; + type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; + description: string; + value: number | string | boolean | DateTime | []; + since?: string; + metadata?: Record; + entityKinds: string[]; + }; +}; + +// @public +export interface FactRetriever { + handler: (ctx: FactRetrieverContext) => Promise; + ref: string; + schema: FactSchema; +} + +// @public +export type FactRetrieverContext = { + config: Config; + discovery: PluginEndpointDiscovery; + logger: Logger_2; +}; + +// @public +export type FactRetrieverRegistration = { + factRetriever: FactRetriever; + cadence?: string; +}; + +// @public +export type FactSchema = { + version: string; + schema: FactValueDefinitions; +}; + +// @public +export type FactValueDefinitions = { + [key: string]: { + type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; + description: string; + since?: string; + metadata?: Record; + entityKinds: string[]; + }; +}; + +// @public +export type FlatTechInsightFact = TechInsightFact & { + ref: string; +}; + +// @public +export interface TechInsightCheck { + // (undocumented) + description: string; + factRefs: string[]; + failureMetadata?: Record; + id: string; + // (undocumented) + name: string; + successMetadata?: Record; +} + +// @public +export interface TechInsightCheckRegistry { + // (undocumented) + get(checkId: string): Promise; + // (undocumented) + getAll(checks: string[]): Promise; + // (undocumented) + list(): Promise; + // (undocumented) + register(check: CheckType): Promise; +} + +// @public +export type TechInsightFact = { + entity: { + namespace: string; + kind: string; + name: string; + }; + facts: Record; + timestamp?: DateTime; +}; + +// @public +export interface TechInsightsStore { + getFactsBetweenTimestampsForRefs( + refs: string[], + entity: string, + startDateTime: DateTime, + endDateTime: DateTime, + ): Promise<{ + [factRef: string]: FlatTechInsightFact[]; + }>; + // (undocumented) + getLatestFactsForRefs( + refs: string[], + entity: string, + ): Promise<{ + [factRef: string]: FlatTechInsightFact; + }>; + getLatestSchemas(refs?: string[]): Promise; + insertFacts(ref: string, facts: TechInsightFact[]): Promise; + insertFactSchema(ref: string, schema: FactSchema): Promise; +} + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/tech-insights-common/package.json b/plugins/tech-insights-common/package.json new file mode 100644 index 0000000000..37d66a9379 --- /dev/null +++ b/plugins/tech-insights-common/package.json @@ -0,0 +1,45 @@ +{ + "name": "@backstage/plugin-tech-insights-common", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/tech-insights-common" + }, + "keywords": [ + "backstage", + "tech-insights" + ], + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/backend-common": "^0.9.0", + "@backstage/config": "^0.1.8", + "@types/luxon": "^2.0.5", + "luxon": "^2.0.2", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/cli": "^0.7.1" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/tech-insights-common/src/checks.ts b/plugins/tech-insights-common/src/checks.ts new file mode 100644 index 0000000000..eaa4792fe3 --- /dev/null +++ b/plugins/tech-insights-common/src/checks.ts @@ -0,0 +1,159 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { TechInsightsStore } from './persistence'; +import { CheckResponse, FactResponse } from './responses'; + +/** + * A factory wrapper to construct FactChecker implementations. + * + * @public + * @typeParam CheckType - Implementation specific Check. Can extend TechInsightCheck with additional information + * @typeParam CheckResultType - Implementation specific result of a check. Can extend CheckResult with additional information + */ +export interface FactCheckerFactory< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + /** + * @param repository - TechInsightsStore + * @returns an implementation of a FactChecker for generic types defined in the factory + */ + construct( + repository: TechInsightsStore, + ): FactChecker; +} + +/** + * FactChecker interface + * + * A generic interface that can be implemented to create checkers for specific check and check return types. + * This is used especially when creating Scorecards and displaying results of rules when run against facts. + * + * @public + * @typeParam CheckType - Implementation specific Check. Can extend TechInsightCheck with additional information + * @typeParam CheckResultType - Implementation specific result of a check. Can extend CheckResult with additional information + */ +export interface FactChecker< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + /** + * Runs checks against an entity. + * + * @param entity - A reference to an entity to run checks against. In a format namespace/kind/name + * @param checks - A collection of checks to run against provided entity + * @returns - A collection containing check/fact information and the actual results of the check + */ + runChecks(entity: string, checks: string[]): Promise; + + /** + * Adds and stores new checks so they can be run checks against. + * Implementation should ideally run validation against the check. + * + * @param check - The actual check to be added. + * @returns - An indicator if fact was successfully added + */ + addCheck(check: CheckType): Promise; + + /** + * Retrieves all available checks that can be used to run checks against. + * The implementation can be just a piping through to CheckRegistry implementation if such is in use. + * + * @returns - A collection of checks + */ + getChecks(): Promise; + + /** + * Validates if check is valid and can be run with the current implementation + * + * @param check - The check to be validated + * @returns - Validation result + */ + validate(check: CheckType): Promise; +} + +/** + * Registry containing checks for tech insights. + * + * @public + * @typeParam CheckType - Implementation specific Check. Can extend TechInsightCheck with additional information + * + */ +export interface TechInsightCheckRegistry { + register(check: CheckType): Promise; + get(checkId: string): Promise; + getAll(checks: string[]): Promise; + list(): Promise; +} + +/** + * Generic CheckResult + * + * Contains information about the facts used to calculate the check result + * and information about the check itself. Both may include metadata to be able to display additional information. + * A collection of these should be parseable by the frontend to display scorecards + * + * @public + */ +export type CheckResult = { + facts: FactResponse; + check: CheckResponse; +}; + +/** + * CheckResult of type Boolean. + * + * @public + */ +export interface BooleanCheckResult extends CheckResult { + result: boolean; +} + +/** + * Generic definition of a check for Tech Insights + * + * @public + */ +export interface TechInsightCheck { + /** + * Unique identifier of the check + * + * Used to identify which checks to use when running checks. + */ + id: string; + + name: string; + description: string; + + /** + * A collection of string referencing fact rows that a check will be run against. + * + * References the fact container, aka fact retriever itself which may or may not contain multiple individual facts and values + */ + factRefs: string[]; + + /** + * Metadata to be returned in case a check has been successfully evaluated + * Can contain links, description texts or other actionable items + */ + successMetadata?: Record; + + /** + * Metadata to be returned in case a check evaluation has ended in failure + * Can contain links, description texts or other actionable items + */ + failureMetadata?: Record; +} diff --git a/plugins/tech-insights-common/src/facts.ts b/plugins/tech-insights-common/src/facts.ts new file mode 100644 index 0000000000..1ae507adf8 --- /dev/null +++ b/plugins/tech-insights-common/src/facts.ts @@ -0,0 +1,196 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { DateTime } from 'luxon'; +import { Config } from '@backstage/config'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Logger } from 'winston'; + +/** + * A container for facts. The shape of the fact records needs to correspond to the FactSchema with same `ref` value. + * Each container contains a reference to an entity which can be a Backstage entity or a generic construct + * outside of backstage with same shape. + * + * Container may contain multiple individual facts and their values + * + * @public + */ +export type TechInsightFact = { + /** + * Entity reference that this fact relates to + */ + entity: { + namespace: string; + kind: string; + name: string; + }; + + /** + * A collection of fact values as key value pairs. + * + * Key indicates fact name as it is defined in FactSchema + */ + facts: Record; + + /** + * Optional timestamp value which can be used to override retrieval time of the fact row. + * Otherwise when stored into data storage, defaults to current time + */ + timestamp?: DateTime; +}; + +/** + + * Response type used when returning from database and API. + * Adds a field for ref for easier usage + * + * @public + */ +export type FlatTechInsightFact = TechInsightFact & { + /** + * Reference and unique identifier of the fact row + */ + ref: string; +}; + +/** + * @public + * + * A record type to specify individual fact shapes + * + * Used as part of a schema to validate, identify and generically construct usage implementations + * of individual fact values in the system. + */ +export type FactValueDefinitions = { + [key: string]: { + /** + * Type of the individual fact value + * + * Numbers are split into integers and floating point values. + * `set` indicates a collection of values + */ + type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; + + /** + * A description of this individual fact value + */ + description: string; + + /** + * Optional semver string to indicate when this specific fact definition was added to the schema + */ + since?: string; + + /** + * Metadata related to an individual fact. + * Can contain links, additional description texts and other actionable data. + * + * Currently loosely typed, but in the future when patterns emerge, key shapes can be defined + * + * examples: + * ``` + * \{ + * link: 'https://sonarqube.mycompany.com/fix-these-issues', + * suggestion: 'To affect this value, you can do x, y, z', + * minValue: 0 + * \} + * ``` + */ + metadata?: Record; + + /** + * A list of entity kind descriptors to indicate if this fact is valid for an entity kind + */ + entityKinds: string[]; + }; +}; + +/** + * @public + * + * Container for FactSchema + */ +export type FactSchema = { + /** + * Semver string indicating the version of this schema + */ + version: string; + /** + * Actual schema definitions for this schema + */ + schema: FactValueDefinitions; +}; + +/** + * @public + * + * FactRetrieverContext injected into individual handler methods of FactRetriever implementations. + * The context can be used to construct logic to retrieve entities, contact integration points + * and fetch and calculate fact values from external sources. + */ +export type FactRetrieverContext = { + config: Config; + discovery: PluginEndpointDiscovery; + logger: Logger; +}; + +/** + * @public + * + * FactRetriever interface + * + * A component specifying + */ +export interface FactRetriever { + /** + * A unique identifier of the retriever. + * Used to identify and store individual facts returned from this retriever + * and schemas defined by this retriever. + */ + ref: string; + + /** + * Handler function that needs to be implemented to retrieve fact values for entities. + * + * @param ctx - FactRetrieverContext which can be used to retrieve config and contact integrations + * @returns - A collection of TechInsightFacts grouped by entities. + */ + handler: (ctx: FactRetrieverContext) => Promise; + + /** + * A fact schema defining the shape of data returned from the handler method for each entity + */ + schema: FactSchema; +} + +/** + * @public + * + * Registration of a fact retriever + * Used to add and schedule individual fact retrievers to the fact retriever engine. + */ +export type FactRetrieverRegistration = { + /** + * Actual FactRetriever implementation + */ + factRetriever: FactRetriever; + + /** + * Cron expression to indicate when the retriever should be triggered. + * Defaults to a random point in time every 24h + * + */ + cadence?: string; +}; diff --git a/plugins/tech-insights-common/src/index.test.ts b/plugins/tech-insights-common/src/index.test.ts new file mode 100644 index 0000000000..0c54118f9a --- /dev/null +++ b/plugins/tech-insights-common/src/index.test.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as anything from './'; + +describe('tech-insights-common', () => { + // Types only + it('should exist', () => { + expect(anything).toBeTruthy(); + }); +}); diff --git a/plugins/tech-insights-common/src/index.ts b/plugins/tech-insights-common/src/index.ts new file mode 100644 index 0000000000..982ae0ac41 --- /dev/null +++ b/plugins/tech-insights-common/src/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './checks'; +export * from './facts'; +export * from './persistence'; +export * from './responses'; diff --git a/plugins/tech-insights-common/src/persistence.ts b/plugins/tech-insights-common/src/persistence.ts new file mode 100644 index 0000000000..9ffb6a9a28 --- /dev/null +++ b/plugins/tech-insights-common/src/persistence.ts @@ -0,0 +1,79 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { FactSchema, TechInsightFact, FlatTechInsightFact } from './facts'; +import { DateTime } from 'luxon'; + +/** + * TechInsights Database + * + * @public + */ +export interface TechInsightsStore { + /** + * Stores fact containers as rows into data store. + * Individual items in array correspond to a fact schema based on reference and entity based on entity identifier. + * + * Each row may contain multiple individual facts and values + * + * @param ref - Unique identifier of the fact retriever these facts relate to + * @param facts - A collection of TechInsightFacts + */ + insertFacts(ref: string, facts: TechInsightFact[]): Promise; + + /** + * @param refs - A collection of reference string to a fact row + * @param entity - A string identifying an entity. In a format namespace/kind/name + * + * @returns - An object keyed by a fact reference and containing an individual TechInsightFact + */ + getLatestFactsForRefs( + refs: string[], + entity: string, + ): Promise<{ [factRef: string]: FlatTechInsightFact }>; + + /** + * Retrieves fact values identified by fact row references for an individual entity. + * + * @param refs - A collection of reference string to a fact row + * @param entity - A string identifying an entity. In a format namespace/kind/name + * @param startDateTime - DateTime object indicating start of the time frame + * @param endDateTime - DateTime object indicating start of the time frame + * + * @returns - An object keyed by a fact reference and containing a collection of TechInsightFacts matching the time frame + */ + getFactsBetweenTimestampsForRefs( + refs: string[], + entity: string, + startDateTime: DateTime, + endDateTime: DateTime, + ): Promise<{ [factRef: string]: FlatTechInsightFact[] }>; + + /** + * Stores versioned fact schemas into data store + * + * @param ref - Identifier of the fact schema. Reference to a fact retriever. + * @param schema - The actual schema to store + */ + insertFactSchema(ref: string, schema: FactSchema): Promise; + + /** + * Retrieves latest versions (as defined by semver) of fact schemas from the data store. + * + * @param refs - Collection of refs to return. If omitted, all Schemas should be returned. + * @returns - A collection of schemas + */ + getLatestSchemas(refs?: string[]): Promise; +} diff --git a/plugins/tech-insights-common/src/responses.ts b/plugins/tech-insights-common/src/responses.ts new file mode 100644 index 0000000000..983a68ee2f --- /dev/null +++ b/plugins/tech-insights-common/src/responses.ts @@ -0,0 +1,100 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DateTime } from 'luxon'; + +/** + * @public + * + * Response type for checks. + */ +export interface CheckResponse { + /** + * Identifier of the Check + */ + id: string; + /** + * Human readable name of the Check + */ + name: string; + /** + * Description of the Check + */ + description: string; + + /** + * A collection of references to fact rows used to run this checks against + */ + factRefs: string[]; + + /** + * Metadata related to a check. + * Can contain links, additional description texts and other actionable data. + * + * Currently loosely typed, but in the future when patterns emerge, key shapes can be defined + */ + metadata?: Record; +} + +/** + * @public + * + * Individual fact response type. + * Keyed by the name of the fact + */ +export type FactResponse = { + [key: string]: { + /** + * Reference and unique identifier of the fact row + */ + ref: string; + /** + * Type of the individual fact value + * + * Numbers are split into integers and floating point values. + * `set` indicates a collection of values + */ + type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; + + /** + * Description of the individual fact + */ + description: string; + + /** + * Actual value of the fact + */ + value: number | string | boolean | DateTime | []; + + /** + * An optional SemVer version identifying when this fact was added to the FactSchema + */ + since?: string; + + /** + * Metadata related to an individual fact. + * Can contain links, additional description texts and other actionable data. + * + * Currently loosely typed, but in the future when patterns emerge, key shapes can be defined + */ + metadata?: Record; + + /** + * A list of entity kind descriptors to indicate if this fact is valid for an entity kind + */ + entityKinds: string[]; + }; +}; diff --git a/plugins/tech-insights-common/src/setupTests.ts b/plugins/tech-insights-common/src/setupTests.ts new file mode 100644 index 0000000000..a330613afb --- /dev/null +++ b/plugins/tech-insights-common/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/yarn.lock b/yarn.lock index c6168a30cf..64f9d5d995 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2172,7 +2172,7 @@ "@babel/helper-validator-identifier" "^7.14.9" to-fast-properties "^2.0.0" -"@backstage/catalog-client@^0.3.18": +"@backstage/catalog-client@^0.3.16", "@backstage/catalog-client@^0.3.18": version "0.3.19" resolved "https://registry.npmjs.org/@backstage/catalog-client/-/catalog-client-0.3.19.tgz#109b0fde1cc2d9a306635f3775fc8f6174172b14" integrity sha512-0ydcC/1ISNgR8SggOUnMNNHtsRwrKN5GX+AXgTVdZk4qpz1MqG1+fzrNld9V7KVvXdsbGDZAl/b5a7/FJEpCqg== @@ -7228,6 +7228,11 @@ resolved "https://registry.npmjs.org/@types/luxon/-/luxon-2.0.4.tgz#f7b5a86ccd843c0ccaddfaedd9ee1081bc1cde3b" integrity sha512-l3xuhmyF2kBldy15SeY6d6HbK2BacEcSK1qTF1ISPtPHr29JH0C1fndz9ExXLKpGl0J6pZi+dGp1i5xesMt60Q== +"@types/luxon@^2.0.5": + version "2.0.5" + resolved "https://registry.npmjs.org/@types/luxon/-/luxon-2.0.5.tgz#29d3b095d55ee50df8f4cf109b16009334d9828e" + integrity sha512-GKrG5v16BOs9XGpouu33hOkAFaiSDi3ZaDXG9F2yAoyzHRBtksZnI60VWY5aM/yAENCccBejrxw8jDY+9OVlxw== + "@types/markdown-to-jsx@^6.11.3": version "6.11.3" resolved "https://registry.npmjs.org/@types/markdown-to-jsx/-/markdown-to-jsx-6.11.3.tgz#cdd1619308fecbc8be7e6a26f3751260249b020e" @@ -7300,6 +7305,18 @@ resolved "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197" integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== +"@types/node-cron@^2.0.4": + version "2.0.5" + resolved "https://registry.npmjs.org/@types/node-cron/-/node-cron-2.0.5.tgz#e244709a86d32453c5a702ced35b53db683fbc8e" + integrity sha512-rQ4kduTmgW11tbtx0/RsoybYHHPu4Vxw5v5ZS5qUKNerlEAI8r8P1F5UUZ2o2HTvzG759sbFxuRuqWxU8zc+EQ== + dependencies: + "@types/tz-offset" "*" + +"@types/node-cron@^3.0.0": + version "3.0.0" + resolved "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.0.tgz#f946cefb5c05c64f460090f6be97bd50460c8898" + integrity sha512-RNBIyVwa/1v2r8/SqK8tadH2sJlFRAo5Ghac/cOcCv4Kp94m0I03UmAh9WVhCqS9ZdB84dF3x47p9aTw8E4c4A== + "@types/node-fetch@^2.5.0", "@types/node-fetch@^2.5.7": version "2.5.8" resolved "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.8.tgz#e199c835d234c7eb0846f6618012e558544ee2fb" @@ -7635,6 +7652,11 @@ resolved "https://registry.npmjs.org/@types/semver/-/semver-6.2.1.tgz#a236185670a7860f1597cf73bea2e16d001461ba" integrity sha512-+beqKQOh9PYxuHvijhVl+tIHvT6tuwOrE9m14zd+MT2A38KoKZhh7pYJ0SNleLtwDsiIxHDsIk9bv01oOxvSvA== +"@types/semver@^7.3.8": + version "7.3.8" + resolved "https://registry.npmjs.org/@types/semver/-/semver-7.3.8.tgz#508a27995498d7586dcecd77c25e289bfaf90c59" + integrity sha512-D/2EJvAlCEtYFEYmmlGwbGXuK886HzyCc3nZX/tkFTQdEU8jZDAgiv08P162yB17y4ZXZoq7yFAnW4GDBb9Now== + "@types/serve-static@*": version "1.13.9" resolved "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.9.tgz#aacf28a85a05ee29a11fb7c3ead935ac56f33e4e" @@ -7806,6 +7828,11 @@ dependencies: "@types/node" "*" +"@types/tz-offset@*": + version "0.0.0" + resolved "https://registry.npmjs.org/@types/tz-offset/-/tz-offset-0.0.0.tgz#d58f1cebd794148d245420f8f0660305d320e565" + integrity sha512-XLD/llTSB6EBe3thkN+/I0L+yCTB6sjrcVovQdx2Cnl6N6bTzHmwe/J8mWnsXFgxLrj/emzdv8IR4evKYG2qxQ== + "@types/uglify-js@*": version "3.0.4" resolved "https://registry.npmjs.org/@types/uglify-js/-/uglify-js-3.0.4.tgz#96beae23df6f561862a830b4288a49e86baac082" @@ -10847,7 +10874,7 @@ clone-stats@^1.0.0: resolved "https://registry.npmjs.org/clone-stats/-/clone-stats-1.0.0.tgz#b3782dff8bb5474e18b9b6bf0fdfe782f8777680" integrity sha1-s3gt/4u1R04Yuba/D9/ngvh3doA= -clone@2.x, clone@^2.1.1: +clone@2.x, clone@^2.1.1, clone@^2.1.2: version "2.1.2" resolved "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f" integrity sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18= @@ -13731,6 +13758,11 @@ eventemitter2@^6.4.3: resolved "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.4.tgz#aa96e8275c4dbeb017a5d0e03780c65612a1202b" integrity sha512-HLU3NDY6wARrLCEwyGKRBvuWYyvW6mHYv72SJJAH3iJN3a6eVUvkjFkcxah1bcTgGVBBrFdIopBJPhCQFMLyXw== +eventemitter2@^6.4.4: + version "6.4.5" + resolved "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.5.tgz#97380f758ae24ac15df8353e0cc27f8b95644655" + integrity sha512-bXE7Dyc1i6oQElDG0jMRZJrRAn9QR2xyyFGmBdZleNmyQX0FqGYmhZIrIrpPfm/w//LTo4tVQGOGQcGCb5q9uw== + eventemitter3@^3.1.0: version "3.1.2" resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7" @@ -15641,6 +15673,11 @@ hash-base@^3.0.0: inherits "^2.0.1" safe-buffer "^5.0.1" +hash-it@^5.0.0: + version "5.0.2" + resolved "https://registry.npmjs.org/hash-it/-/hash-it-5.0.2.tgz#8cc981944964a5124f74f1065af0c0a5d182d556" + integrity sha512-csU3E/a9QEmEgPPxoShVuMcFWM329IGioEPRvYVBv3r5BFrU8pCfnk3jGEVvriAcwqd+nl6KsNhPPjg8MUzkhQ== + hash-stream-validation@^0.2.2: version "0.2.4" resolved "https://registry.npmjs.org/hash-stream-validation/-/hash-stream-validation-0.2.4.tgz#ee68b41bf822f7f44db1142ec28ba9ee7ccb7512" @@ -18010,6 +18047,17 @@ json-pointer@^0.6.0: dependencies: foreach "^2.0.4" +json-rules-engine@^6.1.2: + version "6.1.2" + resolved "https://registry.npmjs.org/json-rules-engine/-/json-rules-engine-6.1.2.tgz#574ef455c10973fd5de07ea8414cbb72bb84d10f" + integrity sha512-+rtKuJ33HAvFywL9broh42FA9hkZNmS0l1DmgjP7nfGJ9E2i2IsfNH0BcXjyXianp/bXAyYlsSv308AfTuvBwQ== + dependencies: + clone "^2.1.2" + eventemitter2 "^6.4.4" + hash-it "^5.0.0" + jsonpath-plus "^5.0.7" + lodash.isobjectlike "^4.0.0" + json-schema-compare@^0.2.2: version "0.2.2" resolved "https://registry.npmjs.org/json-schema-compare/-/json-schema-compare-0.2.2.tgz#dd601508335a90c7f4cfadb6b2e397225c908e56" @@ -18150,6 +18198,11 @@ jsonpath-plus@^0.19.0: resolved "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-0.19.0.tgz#b901e57607055933dc9a8bef0cc25160ee9dd64c" integrity sha512-GSVwsrzW9LsA5lzsqe4CkuZ9wp+kxBb2GwNniaWzI2YFn5Ig42rSW8ZxVpWXaAfakXNrx5pgY5AbQq7kzX29kg== +jsonpath-plus@^5.0.7: + version "5.1.0" + resolved "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-5.1.0.tgz#2fc4b2e461950626c98525425a3a3518b85af6c3" + integrity sha512-890w2Pjtj0iswAxalRlt2kHthi6HKrXEfZcn+ZNZptv7F3rUGIeDuZo+C+h4vXBHLEsVjJrHeCm35nYeZLzSBQ== + jsonpointer@^4.0.1: version "4.1.0" resolved "https://registry.npmjs.org/jsonpointer/-/jsonpointer-4.1.0.tgz#501fb89986a2389765ba09e6053299ceb4f2c2cc" @@ -18934,6 +18987,11 @@ lodash.isobject@^3.0.2: resolved "https://registry.npmjs.org/lodash.isobject/-/lodash.isobject-3.0.2.tgz#3c8fb8d5b5bf4bf90ae06e14f2a530a4ed935e1d" integrity sha1-PI+41bW/S/kK4G4U8qUwpO2TXh0= +lodash.isobjectlike@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/lodash.isobjectlike/-/lodash.isobjectlike-4.0.0.tgz#742c5fc65add27924d3d24191681aa9a17b2b60d" + integrity sha1-dCxfxlrdJ5JNPSQZFoGqmheytg0= + lodash.isplainobject@^4.0.6: version "4.0.6" resolved "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb" @@ -20387,7 +20445,14 @@ modify-values@^1.0.0: resolved "https://registry.npmjs.org/modify-values/-/modify-values-1.0.1.tgz#b3939fa605546474e3e3e3c63d64bd43b4ee6022" integrity sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw== -moment@^2.19.3, moment@^2.27.0, moment@^2.29.1: +moment-timezone@^0.5.31: + version "0.5.33" + resolved "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.33.tgz#b252fd6bb57f341c9b59a5ab61a8e51a73bbd22c" + integrity sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w== + dependencies: + moment ">= 2.9.0" + +"moment@>= 2.9.0", moment@^2.19.3, moment@^2.27.0, moment@^2.29.1: version "2.29.1" resolved "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== @@ -20686,6 +20751,13 @@ node-cache@^5.1.2: dependencies: clone "2.x" +node-cron@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/node-cron/-/node-cron-3.0.0.tgz#b33252803e430f9cd8590cf85738efa1497a9522" + integrity sha512-DDwIvvuCwrNiaU7HEivFDULcaQualDv7KoNlB/UU1wPW0n1tDEmBJKhEIE6DlF2FuoOHcNbLJ8ITL2Iv/3AWmA== + dependencies: + moment-timezone "^0.5.31" + node-dir@^0.1.10, node-dir@^0.1.17: version "0.1.17" resolved "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz#5f5665d93351335caabef8f1c554516cf5f1e4e5" From b2363f3e3b4a54968bc76c1b5429a8fb32ce4d5d Mon Sep 17 00:00:00 2001 From: Jussi Hallila Date: Thu, 21 Oct 2021 09:25:26 +0200 Subject: [PATCH 28/58] Add explicit type information to checks Types can be used to: * Determine which check logic to run * Determine the correct persistence option to use * Choose correct ser/deser logic for the check * Determine correct components to render on the frontend Modify router API to be post for better usability. The API might end up doing writes if caching etc. is added in later as well. Modify CheckRegistry & FactChecker to return persisted check when it is registered. Signed-off-by: Jussi Hallila --- .../README.md | 2 ++ .../api-report.md | 5 ++++- .../src/constants.ts | 20 +++++++++++++++++++ .../src/index.ts | 1 + .../src/service/CheckRegistry.ts | 3 ++- .../JsonRulesEngineFactChecker.test.ts | 9 ++++++++- .../src/service/JsonRulesEngineFactChecker.ts | 19 ++++++++++++++---- .../src/service/router.ts | 17 ++++++++++++---- plugins/tech-insights-common/api-report.md | 8 ++++---- plugins/tech-insights-common/src/checks.ts | 18 +++++++++++++++-- plugins/tech-insights-common/src/responses.ts | 6 ++++++ 11 files changed, 91 insertions(+), 17 deletions(-) create mode 100644 plugins/tech-insights-backend-module-jsonfc/src/constants.ts diff --git a/plugins/tech-insights-backend-module-jsonfc/README.md b/plugins/tech-insights-backend-module-jsonfc/README.md index ea45a1b4b0..99570f1c7a 100644 --- a/plugins/tech-insights-backend-module-jsonfc/README.md +++ b/plugins/tech-insights-backend-module-jsonfc/README.md @@ -52,10 +52,12 @@ Checks for this FactChecker are constructed as `json-rules-engine` compatible JS ```ts import { TechInsightJsonRuleCheck } from '../types'; +import { JSON_RULE_ENGINE_CHECK_TYPE } from '../constants'; export const exampleCheck: TechInsightJsonRuleCheck = { id: 'demodatacheck', // Unique identifier of this check name: 'demodatacheck', // A human readable name of this check to be displayed in the UI + type: JSON_RULE_ENGINE_CHECK_TYPE, // Type identifier of the check. Used to run logic against, determine persistence option to use and render correct components on the UI description: 'A fact check for demoing purposes', // A description to be displayed in the UI factRefs: ['demo-poc.factretriever'], // References to fact containers that this check uses. See documentation on FactRetrievers for more information on these rule: { diff --git a/plugins/tech-insights-backend-module-jsonfc/api-report.md b/plugins/tech-insights-backend-module-jsonfc/api-report.md index bf08028a00..e7c612215c 100644 --- a/plugins/tech-insights-backend-module-jsonfc/api-report.md +++ b/plugins/tech-insights-backend-module-jsonfc/api-report.md @@ -33,6 +33,9 @@ export interface DynamicFact { options?: FactOptions; } +// @public (undocumented) +export const JSON_RULE_ENGINE_CHECK_TYPE = 'json-rules-engine'; + // @public (undocumented) export interface JsonRuleBooleanCheckResult extends BooleanCheckResult { // (undocumented) @@ -60,7 +63,7 @@ export class JsonRulesEngineFactChecker checkRegistry, }: JsonRulesEngineFactCheckerOptions); // (undocumented) - addCheck(check: TechInsightJsonRuleCheck): Promise; + addCheck(check: TechInsightJsonRuleCheck): Promise; // (undocumented) getChecks(): Promise; // (undocumented) diff --git a/plugins/tech-insights-backend-module-jsonfc/src/constants.ts b/plugins/tech-insights-backend-module-jsonfc/src/constants.ts new file mode 100644 index 0000000000..5fd69a3fef --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/constants.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @public + */ +export const JSON_RULE_ENGINE_CHECK_TYPE = 'json-rules-engine'; diff --git a/plugins/tech-insights-backend-module-jsonfc/src/index.ts b/plugins/tech-insights-backend-module-jsonfc/src/index.ts index a2561eaa3b..1a537b8257 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/index.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/index.ts @@ -17,6 +17,7 @@ export { JsonRulesEngineFactCheckerFactory, JsonRulesEngineFactChecker, } from './service/JsonRulesEngineFactChecker'; +export { JSON_RULE_ENGINE_CHECK_TYPE } from './constants'; export type { JsonRulesEngineFactCheckerFactoryOptions, JsonRulesEngineFactCheckerOptions, diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts index 2ae1ac0b83..016fb05b31 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts @@ -31,13 +31,14 @@ export class DefaultCheckRegistry }); } - async register(check: CheckType) { + async register(check: CheckType): Promise { if (this.checks.has(check.id)) { throw new ConflictError( `Tech insight check with id ${check.id} has already been registered`, ); } this.checks.set(check.id, check); + return check; } async get(checkId: string): Promise { diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts index d574e13c5f..549274cefa 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts @@ -18,7 +18,10 @@ import { TechInsightCheckRegistry, TechInsightsStore, } from '@backstage/plugin-tech-insights-common'; -import { JsonRulesEngineFactCheckerFactory } from '../index'; +import { + JSON_RULE_ENGINE_CHECK_TYPE, + JsonRulesEngineFactCheckerFactory, +} from '../index'; import { getVoidLogger } from '@backstage/backend-common'; import { TechInsightJsonRuleCheck } from '../types'; @@ -27,6 +30,7 @@ const testChecks: Record = { { id: 'brokenTestCheck', name: 'brokenTestCheck', + type: JSON_RULE_ENGINE_CHECK_TYPE, description: 'Broken Check For Testing', factRefs: ['test-factretriever'], rule: { @@ -46,6 +50,7 @@ const testChecks: Record = { { id: 'brokenTestCheck2', name: 'brokenTestCheck2', + type: JSON_RULE_ENGINE_CHECK_TYPE, description: 'Second Broken Check For Testing', factRefs: ['non-existing-factretriever'], rule: { @@ -65,6 +70,7 @@ const testChecks: Record = { { id: 'simpleTestCheck', name: 'simpleTestCheck', + type: JSON_RULE_ENGINE_CHECK_TYPE, description: 'Simple Check For Testing', factRefs: ['test-factretriever'], rule: { @@ -85,6 +91,7 @@ const testChecks: Record = { { id: 'simpleTestCheck2', name: 'simpleTestCheck2', + type: JSON_RULE_ENGINE_CHECK_TYPE, description: 'Second Simple Check For Testing', factRefs: ['test-factretriever'], rule: { diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts index 964575b494..cfc31cdffc 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts @@ -29,6 +29,7 @@ import { Logger } from 'winston'; import { pick } from 'lodash'; import Ajv from 'ajv'; import * as validationSchema from './validation-schema.json'; +import { JSON_RULE_ENGINE_CHECK_TYPE } from '../constants'; const noopEvent = { type: 'noop', @@ -113,6 +114,12 @@ export class JsonRulesEngineFactChecker const ajv = new Ajv({ verbose: true }); const validator = ajv.compile(validationSchema); const isValidToSchema = validator(check.rule); + if (check.type !== JSON_RULE_ENGINE_CHECK_TYPE) { + this.logger.warn( + 'Only ${JSON_RULE_ENGINE_CHECK_TYPE} checks can be registered to this fact checker', + ); + return false; + } if (!isValidToSchema) { this.logger.warn( 'Failed to to validate conditions against JSON schema', @@ -146,15 +153,18 @@ export class JsonRulesEngineFactChecker return this.checkRegistry.list(); } - async addCheck(check: TechInsightJsonRuleCheck): Promise { + async addCheck( + check: TechInsightJsonRuleCheck, + ): Promise { if (!(await this.validate(check))) { this.logger.warn( `Check validation failed when adding check ${check.name} to check registry.`, ); - return false; + throw new Error( + 'Failed to add check to rules engine. Validation failed.', + ); } - this.checkRegistry.register(check); - return true; + return await this.checkRegistry.register(check); } private retrieveFactReferences( @@ -216,6 +226,7 @@ export class JsonRulesEngineFactChecker ) { const returnable = { id: techInsightCheck.id, + type: techInsightCheck.type, name: techInsightCheck.name, description: techInsightCheck.description, factRefs: techInsightCheck.factRefs, diff --git a/plugins/tech-insights-backend/src/service/router.ts b/plugins/tech-insights-backend/src/service/router.ts index 3649f522dd..cc41868cd8 100644 --- a/plugins/tech-insights-backend/src/service/router.ts +++ b/plugins/tech-insights-backend/src/service/router.ts @@ -83,11 +83,16 @@ export async function createRouter< return res.send(await factChecker.getChecks()); }); - router.get('/checks/:namespace/:kind/:name', async (req, res) => { + router.post('/checks/run/:namespace/:kind/:name', async (req, res) => { const { namespace, kind, name } = req.params; - const checks = req.query.checks as string[]; - const entityTriplet = `${namespace.toLowerCase()}/${kind.toLowerCase()}/${name.toLowerCase()}`; try { + if (!('checks' in req.body)) { + return res.status(422).send({ + message: 'Failed to get checks from request.', + }); + } + const { checks }: { checks: string[] } = req.body; + const entityTriplet = `${namespace.toLowerCase()}/${kind.toLowerCase()}/${name.toLowerCase()}`; const checkResult = await factChecker.runChecks(entityTriplet, checks); return res.send(checkResult); } catch (e) { @@ -120,7 +125,11 @@ export async function createRouter< const startDatetime = DateTime.fromISO(req.query.startDatetime as string); const endDatetime = DateTime.fromISO(req.query.endDatetime as string); if (!startDatetime.isValid || !endDatetime.isValid) { - return res.status(422).send('Failed to parse datetime from request'); + return res.status(422).send({ + message: 'Failed to parse datetime from request', + field: !startDatetime.isValid ? 'startDateTime' : 'endDateTime', + value: !startDatetime.isValid ? startDatetime : endDatetime, + }); } const entityTriplet = `${namespace.toLowerCase()}/${kind.toLowerCase()}/${name.toLowerCase()}`; return res.send( diff --git a/plugins/tech-insights-common/api-report.md b/plugins/tech-insights-common/api-report.md index 6d0d40239b..40f5203fb8 100644 --- a/plugins/tech-insights-common/api-report.md +++ b/plugins/tech-insights-common/api-report.md @@ -21,6 +21,7 @@ export interface CheckResponse { id: string; metadata?: Record; name: string; + type: string; } // @public @@ -34,7 +35,7 @@ export interface FactChecker< CheckType extends TechInsightCheck, CheckResultType extends CheckResult, > { - addCheck(check: CheckType): Promise; + addCheck(check: CheckType): Promise; getChecks(): Promise; runChecks(entity: string, checks: string[]): Promise; validate(check: CheckType): Promise; @@ -108,14 +109,13 @@ export type FlatTechInsightFact = TechInsightFact & { // @public export interface TechInsightCheck { - // (undocumented) description: string; factRefs: string[]; failureMetadata?: Record; id: string; - // (undocumented) name: string; successMetadata?: Record; + type: string; } // @public @@ -127,7 +127,7 @@ export interface TechInsightCheckRegistry { // (undocumented) list(): Promise; // (undocumented) - register(check: CheckType): Promise; + register(check: CheckType): Promise; } // @public diff --git a/plugins/tech-insights-common/src/checks.ts b/plugins/tech-insights-common/src/checks.ts index eaa4792fe3..02963ab624 100644 --- a/plugins/tech-insights-common/src/checks.ts +++ b/plugins/tech-insights-common/src/checks.ts @@ -66,7 +66,7 @@ export interface FactChecker< * @param check - The actual check to be added. * @returns - An indicator if fact was successfully added */ - addCheck(check: CheckType): Promise; + addCheck(check: CheckType): Promise; /** * Retrieves all available checks that can be used to run checks against. @@ -93,7 +93,7 @@ export interface FactChecker< * */ export interface TechInsightCheckRegistry { - register(check: CheckType): Promise; + register(check: CheckType): Promise; get(checkId: string): Promise; getAll(checks: string[]): Promise; list(): Promise; @@ -135,7 +135,21 @@ export interface TechInsightCheck { */ id: string; + /** + * Type identifier for the check. + * Can be used to determine storage options, logical routing to correct FactChecker implementation + * or to help frontend render correct component types based on this + */ + type: string; + + /** + * Human readable name of the check, may be displayed in the UI + */ name: string; + + /** + * Human readable description of the check, may be displayed in the UI + */ description: string; /** diff --git a/plugins/tech-insights-common/src/responses.ts b/plugins/tech-insights-common/src/responses.ts index 983a68ee2f..0cd1d0a2e8 100644 --- a/plugins/tech-insights-common/src/responses.ts +++ b/plugins/tech-insights-common/src/responses.ts @@ -26,6 +26,12 @@ export interface CheckResponse { * Identifier of the Check */ id: string; + /** + * Type identifier for the check. + * Can be used to determine storage options, logical routing to correct FactChecker implementation + * or to help frontend render correct component types based on this + */ + type: string; /** * Human readable name of the Check */ From b427f4ba0b4b05f167f2cb261d9e83c789043207 Mon Sep 17 00:00:00 2001 From: Jussi Hallila Date: Thu, 21 Oct 2021 11:48:00 +0200 Subject: [PATCH 29/58] Modify validation to have a more user friendly API * Adding a container type to be returned from validate function * Modifying JsonRulesEngineFactChecker to throw with error information if check addition fails on validation Signed-off-by: Jussi Hallila --- .../api-report.md | 3 +- .../JsonRulesEngineFactChecker.test.ts | 12 +++-- .../src/service/JsonRulesEngineFactChecker.ts | 50 +++++++++++++------ plugins/tech-insights-common/api-report.md | 25 +++++++++- plugins/tech-insights-common/package.json | 1 + plugins/tech-insights-common/src/checks.ts | 39 ++++++++++++++- 6 files changed, 109 insertions(+), 21 deletions(-) diff --git a/plugins/tech-insights-backend-module-jsonfc/api-report.md b/plugins/tech-insights-backend-module-jsonfc/api-report.md index e7c612215c..8f6bf0cd6c 100644 --- a/plugins/tech-insights-backend-module-jsonfc/api-report.md +++ b/plugins/tech-insights-backend-module-jsonfc/api-report.md @@ -5,6 +5,7 @@ ```ts import { BooleanCheckResult } from '@backstage/plugin-tech-insights-common'; import { CheckResponse } from '@backstage/plugin-tech-insights-common'; +import { CheckValidationResponse } from '@backstage/plugin-tech-insights-common'; import { DynamicFactCallback } from 'json-rules-engine'; import { FactChecker } from '@backstage/plugin-tech-insights-common'; import { FactOptions } from 'json-rules-engine'; @@ -72,7 +73,7 @@ export class JsonRulesEngineFactChecker checks: string[], ): Promise; // (undocumented) - validate(check: TechInsightJsonRuleCheck): Promise; + validate(check: TechInsightJsonRuleCheck): Promise; } // @public diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts index 549274cefa..124030a545 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts @@ -289,12 +289,16 @@ describe('JsonRulesEngineFactChecker', () => { describe('when validating checks', () => { it('should succeed on valid rules', async () => { - const isValid = await factChecker.validate(testChecks.simple[0]); - expect(isValid).toBeTruthy(); + const validationResponse = await factChecker.validate( + testChecks.simple[0], + ); + expect(validationResponse.valid).toBeTruthy(); }); it('should fail on broken rules', async () => { - const isValid = await factChecker.validate(testChecks.broken[0]); - expect(isValid).toBeFalsy(); + const validationResponse = await factChecker.validate( + testChecks.broken[0], + ); + expect(validationResponse.valid).toBeFalsy(); }); }); }); diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts index cfc31cdffc..7e8cab8b7f 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts @@ -22,6 +22,8 @@ import { TechInsightCheckRegistry, FlatTechInsightFact, TechInsightsStore, + CheckValidationResponse, + CheckValidationError, } from '@backstage/plugin-tech-insights-common'; import { Engine, EngineResult, TopLevelCondition } from 'json-rules-engine'; import { DefaultCheckRegistry } from './CheckRegistry'; @@ -110,22 +112,31 @@ export class JsonRulesEngineFactChecker } } - async validate(check: TechInsightJsonRuleCheck): Promise { + async validate( + check: TechInsightJsonRuleCheck, + ): Promise { const ajv = new Ajv({ verbose: true }); const validator = ajv.compile(validationSchema); const isValidToSchema = validator(check.rule); if (check.type !== JSON_RULE_ENGINE_CHECK_TYPE) { - this.logger.warn( - 'Only ${JSON_RULE_ENGINE_CHECK_TYPE} checks can be registered to this fact checker', - ); - return false; + const msg = `Only ${JSON_RULE_ENGINE_CHECK_TYPE} checks can be registered to this fact checker`; + this.logger.warn(msg); + return { + valid: false, + message: msg, + }; } if (!isValidToSchema) { + const msg = 'Failed to to validate conditions against JSON schema'; this.logger.warn( 'Failed to to validate conditions against JSON schema', validator.errors, ); - return false; + return { + valid: false, + message: msg, + errors: validator.errors, + }; } const existingSchemas = await this.repository.getLatestSchemas( @@ -146,7 +157,17 @@ export class JsonRulesEngineFactChecker )}`, ); }); - return failedReferences.length === 0; + const valid = failedReferences.length === 0; + return { + valid, + ...(!valid + ? { + message: `Check is referencing missing values from fact schemas: ${failedReferences + .map(it => it.ref) + .join(',')}`, + } + : {}), + }; } getChecks(): Promise { @@ -156,13 +177,14 @@ export class JsonRulesEngineFactChecker async addCheck( check: TechInsightJsonRuleCheck, ): Promise { - if (!(await this.validate(check))) { - this.logger.warn( - `Check validation failed when adding check ${check.name} to check registry.`, - ); - throw new Error( - 'Failed to add check to rules engine. Validation failed.', - ); + const checkValidationResponse = await this.validate(check); + if (!checkValidationResponse.valid) { + const msg = `Check validation failed when adding check ${check.name} to check registry.`; + this.logger.warn(msg); + throw new CheckValidationError({ + message: checkValidationResponse.message || msg, + errors: checkValidationResponse.errors, + }); } return await this.checkRegistry.register(check); } diff --git a/plugins/tech-insights-common/api-report.md b/plugins/tech-insights-common/api-report.md index 40f5203fb8..3c599870c5 100644 --- a/plugins/tech-insights-common/api-report.md +++ b/plugins/tech-insights-common/api-report.md @@ -4,6 +4,7 @@ ```ts import { Config } from '@backstage/config'; +import { CustomErrorBase } from '@backstage/errors'; import { DateTime } from 'luxon'; import { Logger as Logger_2 } from 'winston'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; @@ -30,6 +31,28 @@ export type CheckResult = { check: CheckResponse; }; +// @public +export class CheckValidationError extends CustomErrorBase { + constructor({ + message, + cause, + errors, + }: { + message: string; + cause?: Error; + errors?: any; + }); + // (undocumented) + errors?: any; +} + +// @public +export type CheckValidationResponse = { + valid: boolean; + message?: string; + errors?: any; +}; + // @public export interface FactChecker< CheckType extends TechInsightCheck, @@ -38,7 +61,7 @@ export interface FactChecker< addCheck(check: CheckType): Promise; getChecks(): Promise; runChecks(entity: string, checks: string[]): Promise; - validate(check: CheckType): Promise; + validate(check: CheckType): Promise; } // @public diff --git a/plugins/tech-insights-common/package.json b/plugins/tech-insights-common/package.json index 37d66a9379..15ccc1a682 100644 --- a/plugins/tech-insights-common/package.json +++ b/plugins/tech-insights-common/package.json @@ -31,6 +31,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.9.0", + "@backstage/errors": "^0.1.2", "@backstage/config": "^0.1.8", "@types/luxon": "^2.0.5", "luxon": "^2.0.2", diff --git a/plugins/tech-insights-common/src/checks.ts b/plugins/tech-insights-common/src/checks.ts index 02963ab624..062a5baaaa 100644 --- a/plugins/tech-insights-common/src/checks.ts +++ b/plugins/tech-insights-common/src/checks.ts @@ -15,6 +15,7 @@ */ import { TechInsightsStore } from './persistence'; import { CheckResponse, FactResponse } from './responses'; +import { CustomErrorBase } from '@backstage/errors'; /** * A factory wrapper to construct FactChecker implementations. @@ -82,7 +83,7 @@ export interface FactChecker< * @param check - The check to be validated * @returns - Validation result */ - validate(check: CheckType): Promise; + validate(check: CheckType): Promise; } /** @@ -171,3 +172,39 @@ export interface TechInsightCheck { */ failureMetadata?: Record; } + +/** + * Validation response from CheckValidator + * + * May contain additional data for display purposes + * @public + */ +export type CheckValidationResponse = { + valid: boolean; + message?: string; + errors?: any; +}; + +/** + * Error object for Validation Errors + * + * Can be used to short circuit larger execution paths instead of passing validation response around + * + * @public + */ +export class CheckValidationError extends CustomErrorBase { + errors?: any; + + constructor({ + message, + cause, + errors, + }: { + message: string; + cause?: Error; + errors?: any; + }) { + super(message, cause); + this.errors = errors; + } +} From a1afbe0498aa4a0982efe9b448d94a59d9f419aa Mon Sep 17 00:00:00 2001 From: Jussi Hallila Date: Thu, 21 Oct 2021 14:37:55 +0200 Subject: [PATCH 30/58] Use a direct custom Error since backstage/errors seems unusable. Signed-off-by: Jussi Hallila --- plugins/tech-insights-common/api-report.md | 13 ++----------- plugins/tech-insights-common/package.json | 1 - plugins/tech-insights-common/src/checks.ts | 15 +++------------ 3 files changed, 5 insertions(+), 24 deletions(-) diff --git a/plugins/tech-insights-common/api-report.md b/plugins/tech-insights-common/api-report.md index 3c599870c5..db987546c3 100644 --- a/plugins/tech-insights-common/api-report.md +++ b/plugins/tech-insights-common/api-report.md @@ -4,7 +4,6 @@ ```ts import { Config } from '@backstage/config'; -import { CustomErrorBase } from '@backstage/errors'; import { DateTime } from 'luxon'; import { Logger as Logger_2 } from 'winston'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; @@ -32,16 +31,8 @@ export type CheckResult = { }; // @public -export class CheckValidationError extends CustomErrorBase { - constructor({ - message, - cause, - errors, - }: { - message: string; - cause?: Error; - errors?: any; - }); +export class CheckValidationError extends Error { + constructor({ message, errors }: { message: string; errors?: any }); // (undocumented) errors?: any; } diff --git a/plugins/tech-insights-common/package.json b/plugins/tech-insights-common/package.json index 15ccc1a682..37d66a9379 100644 --- a/plugins/tech-insights-common/package.json +++ b/plugins/tech-insights-common/package.json @@ -31,7 +31,6 @@ }, "dependencies": { "@backstage/backend-common": "^0.9.0", - "@backstage/errors": "^0.1.2", "@backstage/config": "^0.1.8", "@types/luxon": "^2.0.5", "luxon": "^2.0.2", diff --git a/plugins/tech-insights-common/src/checks.ts b/plugins/tech-insights-common/src/checks.ts index 062a5baaaa..7b2c4c6b8f 100644 --- a/plugins/tech-insights-common/src/checks.ts +++ b/plugins/tech-insights-common/src/checks.ts @@ -15,7 +15,6 @@ */ import { TechInsightsStore } from './persistence'; import { CheckResponse, FactResponse } from './responses'; -import { CustomErrorBase } from '@backstage/errors'; /** * A factory wrapper to construct FactChecker implementations. @@ -192,19 +191,11 @@ export type CheckValidationResponse = { * * @public */ -export class CheckValidationError extends CustomErrorBase { +export class CheckValidationError extends Error { errors?: any; - constructor({ - message, - cause, - errors, - }: { - message: string; - cause?: Error; - errors?: any; - }) { - super(message, cause); + constructor({ message, errors }: { message: string; errors?: any }) { + super(message); this.errors = errors; } } From df000b95969d74b8d264543110989283650e05d6 Mon Sep 17 00:00:00 2001 From: Jussi Hallila Date: Mon, 25 Oct 2021 17:04:33 +0200 Subject: [PATCH 31/58] Address various code review comments. Signed-off-by: Jussi Hallila --- .changeset/bright-pandas-rush.md | 17 --- packages/backend/src/index.ts | 2 +- packages/backend/src/plugins/techInsights.ts | 71 +++++++++- .../CHANGELOG.md | 7 - .../README.md | 2 +- .../api-report.md | 14 -- .../package.json | 4 +- .../src/index.ts | 1 - .../JsonRulesEngineFactChecker.test.ts | 99 +++++++------- .../src/service/JsonRulesEngineFactChecker.ts | 46 +++---- .../src/types.ts | 16 +-- plugins/tech-insights-backend/CHANGELOG.md | 7 - plugins/tech-insights-backend/README.md | 2 +- plugins/tech-insights-backend/api-report.md | 19 +-- .../migrations/202109061111_fact_schemas.js | 17 ++- .../migrations/202109061212_facts.js | 21 ++- plugins/tech-insights-backend/src/index.ts | 4 +- .../service/fact/FactRetrieverEngine.test.ts | 45 +++---- .../src/service/fact/FactRetrieverEngine.ts | 22 +-- .../src/service/fact/FactRetrieverRegistry.ts | 8 +- .../persistence/TechInsightsDatabase.test.ts | 127 ++++++++++++------ .../persistence/TechInsightsDatabase.ts | 94 ++++++------- .../src/service/router.test.ts | 51 +++---- .../src/service/router.ts | 44 ++++-- ...ilder.ts => techInsightsContextBuilder.ts} | 91 ++++++------- plugins/tech-insights-common/CHANGELOG.md | 7 - plugins/tech-insights-common/api-report.md | 55 ++++---- plugins/tech-insights-common/src/checks.ts | 2 +- plugins/tech-insights-common/src/facts.ts | 64 +++++---- .../tech-insights-common/src/persistence.ts | 32 +++-- plugins/tech-insights-common/src/responses.ts | 11 +- 31 files changed, 529 insertions(+), 473 deletions(-) delete mode 100644 .changeset/bright-pandas-rush.md delete mode 100644 plugins/tech-insights-backend-module-jsonfc/CHANGELOG.md delete mode 100644 plugins/tech-insights-backend/CHANGELOG.md rename plugins/tech-insights-backend/src/service/{DefaultTechInsightsBuilder.ts => techInsightsContextBuilder.ts} (63%) delete mode 100644 plugins/tech-insights-common/CHANGELOG.md diff --git a/.changeset/bright-pandas-rush.md b/.changeset/bright-pandas-rush.md deleted file mode 100644 index 86b3b819ee..0000000000 --- a/.changeset/bright-pandas-rush.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -'@backstage/plugin-tech-insights-backend': minor -'@backstage/plugin-tech-insights-backend-module-jsonfc': minor -'@backstage/plugin-tech-insights-common': minor ---- - -## Add initial implementation of Tech Insights backend - -Add common types and interfaces for Tech Insights. Exposing components needed to use and modify tech-insights-backend module and to implement individual Fact Retrievers, Fact Checkers and their persistence options. - -Implement a framework to run fact retrievers and store fact data into the database. Add migration scripts to create a new database for `tech_insights`. - -Create a default implementation of a FactChecker enabling users to construct checks and run them and generate scorecards based on checks. - -To be able to use tech insights in your application you need to implement `FactRetriever`s to retrieve and return data for facts and register them to the tech-insights-backend. If you want to use fact checking functionality, you need to create `check`s and register them to an implementation of a `FactChecker`. - -For more information see documentation on the README.md files of the respective packages. diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 6d40072f99..ffdce949b7 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -109,7 +109,7 @@ async function main() { const badgesEnv = useHotMemoize(module, () => createEnv('badges')); const jenkinsEnv = useHotMemoize(module, () => createEnv('jenkins')); const techInsightsEnv = useHotMemoize(module, () => - createEnv('tech_insights'), + createEnv('tech-insights'), ); const apiRouter = Router(); diff --git a/packages/backend/src/plugins/techInsights.ts b/packages/backend/src/plugins/techInsights.ts index c396f4b82b..d51c0c7575 100644 --- a/packages/backend/src/plugins/techInsights.ts +++ b/packages/backend/src/plugins/techInsights.ts @@ -15,11 +15,15 @@ */ import { createRouter, - DefaultTechInsightsBuilder, + buildTechInsightsContext, } from '@backstage/plugin-tech-insights-backend'; import { Router } from 'express'; import { PluginEnvironment } from '../types'; -import { JsonRulesEngineFactCheckerFactory } from '@backstage/plugin-tech-insights-backend-module-jsonfc'; +import { + JSON_RULE_ENGINE_CHECK_TYPE, + JsonRulesEngineFactCheckerFactory, +} from '@backstage/plugin-tech-insights-backend-module-jsonfc'; +import { CatalogClient } from '@backstage/catalog-client'; export default async function createPlugin({ logger, @@ -27,20 +31,75 @@ export default async function createPlugin({ discovery, database, }: PluginEnvironment): Promise { - const builder = new DefaultTechInsightsBuilder({ + const techInsightsContext = await buildTechInsightsContext({ logger, config, database, discovery, - factRetrievers: [], + factRetrievers: [ + { + cadence: '1 1 1 * *', // At 01:01 on day-of-month 1. + factRetriever: { + id: 'testRetriever', + version: '1.1.1', + entityTypes: ['component'], + schema: { + examplenumberfact: { + type: 'integer', + description: '', + }, + }, + handler: async _ctx => { + const catalogClient = new CatalogClient({ + discoveryApi: discovery, + }); + const entities = await catalogClient.getEntities(); + + return Promise.resolve( + entities.items.map(it => { + return { + entity: { + namespace: it.metadata.namespace!!, + kind: it.kind, + name: it.metadata.name, + }, + facts: { + examplenumberfact: 2, + }, + }; + }), + ); + }, + }, + }, + ], factCheckerFactory: new JsonRulesEngineFactCheckerFactory({ - checks: [], + checks: [ + { + id: 'simpleTestCheck', + type: JSON_RULE_ENGINE_CHECK_TYPE, + name: 'simpleTestCheck', + description: 'Simple Check For Testing', + factIds: ['testRetriever'], + rule: { + conditions: { + all: [ + { + fact: 'examplenumberfact', + operator: 'lessThan', + value: 5, + }, + ], + }, + }, + }, + ], logger, }), }); return await createRouter({ - ...(await builder.build()), + ...techInsightsContext, logger, config, }); diff --git a/plugins/tech-insights-backend-module-jsonfc/CHANGELOG.md b/plugins/tech-insights-backend-module-jsonfc/CHANGELOG.md deleted file mode 100644 index d0aced5bf5..0000000000 --- a/plugins/tech-insights-backend-module-jsonfc/CHANGELOG.md +++ /dev/null @@ -1,7 +0,0 @@ -# @backstage/plugin-tech-insights-backend-module-jsonfc - -## 0.0.1 - -### Patch Changes - -- Initial implementation diff --git a/plugins/tech-insights-backend-module-jsonfc/README.md b/plugins/tech-insights-backend-module-jsonfc/README.md index 99570f1c7a..1404bd4ea4 100644 --- a/plugins/tech-insights-backend-module-jsonfc/README.md +++ b/plugins/tech-insights-backend-module-jsonfc/README.md @@ -2,7 +2,7 @@ This is an extension to module to tech-insights-backend plugin, which provides basic framework and functionality to implement tech insights within Backstage. -This module provides functionality to run checks against a `json-rules-engine` and provide boolean logic by simply building checks using JSON conditions. +This module provides functionality to run checks against a [json-rules-engine](https://github.com/CacheControl/json-rules-engine) and provide boolean logic by simply building checks using JSON conditions. ## Getting started diff --git a/plugins/tech-insights-backend-module-jsonfc/api-report.md b/plugins/tech-insights-backend-module-jsonfc/api-report.md index 8f6bf0cd6c..3d214566c5 100644 --- a/plugins/tech-insights-backend-module-jsonfc/api-report.md +++ b/plugins/tech-insights-backend-module-jsonfc/api-report.md @@ -6,9 +6,7 @@ import { BooleanCheckResult } from '@backstage/plugin-tech-insights-common'; import { CheckResponse } from '@backstage/plugin-tech-insights-common'; import { CheckValidationResponse } from '@backstage/plugin-tech-insights-common'; -import { DynamicFactCallback } from 'json-rules-engine'; import { FactChecker } from '@backstage/plugin-tech-insights-common'; -import { FactOptions } from 'json-rules-engine'; import { Logger as Logger_2 } from 'winston'; import { TechInsightCheck } from '@backstage/plugin-tech-insights-common'; import { TechInsightCheckRegistry } from '@backstage/plugin-tech-insights-common'; @@ -24,16 +22,6 @@ export type CheckCondition = { result: boolean; }; -// @public (undocumented) -export interface DynamicFact { - // (undocumented) - calculationMethod: DynamicFactCallback | T; - // (undocumented) - id: string; - // (undocumented) - options?: FactOptions; -} - // @public (undocumented) export const JSON_RULE_ENGINE_CHECK_TYPE = 'json-rules-engine'; @@ -120,8 +108,6 @@ export type Rule = { // @public (undocumented) export interface TechInsightJsonRuleCheck extends TechInsightCheck { - // (undocumented) - dynamicFacts?: DynamicFact[]; // (undocumented) rule: Rule; } diff --git a/plugins/tech-insights-backend-module-jsonfc/package.json b/plugins/tech-insights-backend-module-jsonfc/package.json index 945947dd3b..c11f20ee38 100644 --- a/plugins/tech-insights-backend-module-jsonfc/package.json +++ b/plugins/tech-insights-backend-module-jsonfc/package.json @@ -1,8 +1,8 @@ { "name": "@backstage/plugin-tech-insights-backend-module-jsonfc", "version": "0.1.0", - "main": "src/index.ts", - "types": "src/index.ts", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts", "license": "Apache-2.0", "private": false, "publishConfig": { diff --git a/plugins/tech-insights-backend-module-jsonfc/src/index.ts b/plugins/tech-insights-backend-module-jsonfc/src/index.ts index 1a537b8257..1239211f73 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/index.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/index.ts @@ -28,6 +28,5 @@ export type { TechInsightJsonRuleCheck, ResponseTopLevelCondition, Rule, - DynamicFact, CheckCondition, } from './types'; diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts index 124030a545..a31ae90c70 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts @@ -32,7 +32,7 @@ const testChecks: Record = { name: 'brokenTestCheck', type: JSON_RULE_ENGINE_CHECK_TYPE, description: 'Broken Check For Testing', - factRefs: ['test-factretriever'], + factIds: ['test-factretriever'], rule: { conditions: { all: [ @@ -52,7 +52,7 @@ const testChecks: Record = { name: 'brokenTestCheck2', type: JSON_RULE_ENGINE_CHECK_TYPE, description: 'Second Broken Check For Testing', - factRefs: ['non-existing-factretriever'], + factIds: ['non-existing-factretriever'], rule: { conditions: { any: [ @@ -72,7 +72,7 @@ const testChecks: Record = { name: 'simpleTestCheck', type: JSON_RULE_ENGINE_CHECK_TYPE, description: 'Simple Check For Testing', - factRefs: ['test-factretriever'], + factIds: ['test-factretriever'], rule: { conditions: { all: [ @@ -93,7 +93,7 @@ const testChecks: Record = { name: 'simpleTestCheck2', type: JSON_RULE_ENGINE_CHECK_TYPE, description: 'Second Simple Check For Testing', - factRefs: ['test-factretriever'], + factIds: ['test-factretriever'], rule: { conditions: { all: [ @@ -114,17 +114,15 @@ const latestSchemasMock = jest.fn().mockImplementation(() => [ version: '0.0.1', id: 2, ref: 'test-factretriever', - schema: { - testnumberfact: { - type: 'integer', - description: '', - entityKinds: ['component'], - }, + entityTypes: ['component'], + testnumberfact: { + type: 'integer', + description: '', }, }, ]); -const factsBetweenTimestampsForRefsMock = jest.fn(); -const latestFactsForRefsMock = jest.fn().mockImplementation(() => ({})); +const factsBetweenTimestampsByIdsMock = jest.fn(); +const latestFactsByIdsMock = jest.fn().mockImplementation(() => ({})); const mockCheckRegistry = { getAll(checks: string[]) { return checks.flatMap(check => testChecks[check]); @@ -132,8 +130,8 @@ const mockCheckRegistry = { } as unknown as TechInsightCheckRegistry; const mockRepository: TechInsightsStore = { - getLatestFactsForRefs: latestFactsForRefsMock, - getFactsBetweenTimestampsForRefs: factsBetweenTimestampsForRefsMock, + getLatestFactsByIds: latestFactsByIdsMock, + getFactsBetweenTimestampsByIds: factsBetweenTimestampsByIdsMock, getLatestSchemas: latestSchemasMock, } as unknown as TechInsightsStore; @@ -159,10 +157,10 @@ describe('JsonRulesEngineFactChecker', () => { ); }); it('should respond with result, facts, fact schemas and checks', async () => { - latestFactsForRefsMock.mockImplementation(() => + latestFactsByIdsMock.mockImplementation(() => Promise.resolve({ ['test-factretriever']: { - ref: 'test-factretriever', + id: 'test-factretriever', facts: { testnumberfact: 3, }, @@ -170,46 +168,45 @@ describe('JsonRulesEngineFactChecker', () => { }), ); const results = await factChecker.runChecks('a/a/a', ['simple']); - expect(results).toMatchObject([ - { - facts: { - testnumberfact: { - value: 3, - type: 'integer', - description: '', - entityKinds: ['component'], - }, + expect(results).toHaveLength(1); + expect(results[0]).toMatchObject({ + facts: { + testnumberfact: { + value: 3, + type: 'integer', + description: '', }, - result: true, - check: { - id: 'simpleTestCheck', - name: 'simpleTestCheck', - description: 'Simple Check For Testing', - factRefs: ['test-factretriever'], - rule: { - conditions: { - all: [ - { - fact: 'testnumberfact', - factResult: 3, - operator: 'lessThan', - result: true, - value: 5, - }, - ], - priority: 1, - }, + }, + result: true, + check: { + id: 'simpleTestCheck', + type: JSON_RULE_ENGINE_CHECK_TYPE, + name: 'simpleTestCheck', + description: 'Simple Check For Testing', + factIds: ['test-factretriever'], + rule: { + conditions: { + all: [ + { + fact: 'testnumberfact', + factResult: 3, + operator: 'lessThan', + result: true, + value: 5, + }, + ], + priority: 1, }, }, }, - ]); + }); }); it('should gracefully handle multiple check at once', async () => { - latestFactsForRefsMock.mockImplementation(() => + latestFactsByIdsMock.mockImplementation(() => Promise.resolve({ ['test-factretriever']: { - ref: 'test-factretriever', + id: 'test-factretriever', facts: { testnumberfact: 3, }, @@ -227,15 +224,15 @@ describe('JsonRulesEngineFactChecker', () => { value: 3, type: 'integer', description: '', - entityKinds: ['component'], }, }, result: true, check: { id: 'simpleTestCheck', + type: JSON_RULE_ENGINE_CHECK_TYPE, name: 'simpleTestCheck', description: 'Simple Check For Testing', - factRefs: ['test-factretriever'], + factIds: ['test-factretriever'], rule: { conditions: { priority: 1, @@ -258,15 +255,15 @@ describe('JsonRulesEngineFactChecker', () => { value: 3, type: 'integer', description: '', - entityKinds: ['component'], }, }, result: true, check: { id: 'simpleTestCheck2', + type: JSON_RULE_ENGINE_CHECK_TYPE, name: 'simpleTestCheck2', description: 'Second Simple Check For Testing', - factRefs: ['test-factretriever'], + factIds: ['test-factretriever'], rule: { conditions: { priority: 1, diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts index 7e8cab8b7f..ab90a3d828 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts @@ -18,7 +18,6 @@ import { JsonRuleBooleanCheckResult, TechInsightJsonRuleCheck } from '../types'; import { FactChecker, FactResponse, - FactValueDefinitions, TechInsightCheckRegistry, FlatTechInsightFact, TechInsightsStore, @@ -81,20 +80,13 @@ export class JsonRulesEngineFactChecker ): Promise { const engine = new Engine(); const techInsightChecks = await this.checkRegistry.getAll(checks); - const factRefs = techInsightChecks.flatMap(it => it.factRefs); - const facts = await this.repository.getLatestFactsForRefs(factRefs, entity); + const factIds = techInsightChecks.flatMap(it => it.factIds); + const facts = await this.repository.getLatestFactsByIds(factIds, entity); techInsightChecks.forEach(techInsightCheck => { const rule = techInsightCheck.rule; rule.name = techInsightCheck.id; engine.addRule({ ...techInsightCheck.rule, event: noopEvent }); - - if (techInsightCheck.dynamicFacts) { - techInsightCheck.dynamicFacts.forEach(it => - engine.addFact(it.id, it.calculationMethod, it.options), - ); - } }); - const factValues = Object.values(facts).reduce( (acc, it) => ({ ...acc, ...it.facts }), {}, @@ -140,21 +132,21 @@ export class JsonRulesEngineFactChecker } const existingSchemas = await this.repository.getLatestSchemas( - check.factRefs, + check.factIds, + ); + const references = this.retrieveIndividualFactReferences( + check.rule.conditions, ); - const references = this.retrieveFactReferences(check.rule.conditions); const results = references.map(ref => ({ ref, - result: existingSchemas.some(schema => schema.schema.hasOwnProperty(ref)), + result: existingSchemas.some(schema => schema.hasOwnProperty(ref)), })); const failedReferences = results.filter(it => !it.result); failedReferences.forEach(it => { this.logger.warn( `Validation failed for check ${check.name}. Reference to value ${ it.ref - } does not exists in referred fact schemas: ${check.factRefs.join( - ',', - )}`, + } does not exists in referred fact schemas: ${check.factIds.join(',')}`, ); }); const valid = failedReferences.length === 0; @@ -189,17 +181,21 @@ export class JsonRulesEngineFactChecker return await this.checkRegistry.register(check); } - private retrieveFactReferences( + private retrieveIndividualFactReferences( condition: TopLevelCondition | { fact: string }, ): string[] { let results: string[] = []; if ('all' in condition) { results = results.concat( - condition.all.flatMap(con => this.retrieveFactReferences(con)), + condition.all.flatMap(con => + this.retrieveIndividualFactReferences(con), + ), ); } else if ('any' in condition) { results = results.concat( - condition.any.flatMap(con => this.retrieveFactReferences(con)), + condition.any.flatMap(con => + this.retrieveIndividualFactReferences(con), + ), ); } else { results.push(condition.fact); @@ -251,7 +247,7 @@ export class JsonRulesEngineFactChecker type: techInsightCheck.type, name: techInsightCheck.name, description: techInsightCheck.description, - factRefs: techInsightCheck.factRefs, + factIds: techInsightCheck.factIds, metadata: result.result ? techInsightCheck.successMetadata : techInsightCheck.failureMetadata, @@ -272,18 +268,18 @@ export class JsonRulesEngineFactChecker techInsightCheck: TechInsightJsonRuleCheck, ): Promise { const factSchemas = await this.repository.getLatestSchemas( - techInsightCheck.factRefs, + techInsightCheck.factIds, ); - const schemas: FactValueDefinitions = factSchemas.reduce( - (acc, schema) => ({ ...acc, ...schema.schema }), + const schemas = factSchemas.reduce( + (acc, schema) => ({ ...acc, ...schema }), {}, ); - const individualFacts = this.retrieveFactReferences( + const individualFacts = this.retrieveIndividualFactReferences( techInsightCheck.rule.conditions, ); const factValues = facts .filter(factContainer => - techInsightCheck.factRefs.includes(factContainer.ref), + techInsightCheck.factIds.includes(factContainer.id), ) .reduce( (acc, factContainer) => ({ diff --git a/plugins/tech-insights-backend-module-jsonfc/src/types.ts b/plugins/tech-insights-backend-module-jsonfc/src/types.ts index 5f6210d110..23d5849eda 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/types.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/types.ts @@ -13,26 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { - DynamicFactCallback, - FactOptions, - TopLevelCondition, -} from 'json-rules-engine'; +import { TopLevelCondition } from 'json-rules-engine'; import { BooleanCheckResult, CheckResponse, TechInsightCheck, } from '@backstage/plugin-tech-insights-common'; -/** - * @public - */ -export interface DynamicFact { - id: string; - calculationMethod: DynamicFactCallback | T; - options?: FactOptions; -} - /** * @public */ @@ -47,7 +34,6 @@ export type Rule = { */ export interface TechInsightJsonRuleCheck extends TechInsightCheck { rule: Rule; - dynamicFacts?: DynamicFact[]; } /** diff --git a/plugins/tech-insights-backend/CHANGELOG.md b/plugins/tech-insights-backend/CHANGELOG.md deleted file mode 100644 index 17e0211ab0..0000000000 --- a/plugins/tech-insights-backend/CHANGELOG.md +++ /dev/null @@ -1,7 +0,0 @@ -# @backstage/plugin-tech-insights-backend - -## 0.0.1 - -### Patch Changes - -- Initial implementation diff --git a/plugins/tech-insights-backend/README.md b/plugins/tech-insights-backend/README.md index 4e5b5ad1b3..32265c1ced 100644 --- a/plugins/tech-insights-backend/README.md +++ b/plugins/tech-insights-backend/README.md @@ -124,7 +124,7 @@ const myFactRetriever: FactRetriever = { examplenumberfact: { type: 'integer', // Type of the fact description: 'A fact of a number', // Description of the fact - entityKinds: ['component'], // An array of entity kinds that this fact is applicable to + entityTypes: ['component'], // An array of entity kinds that this fact is applicable to }, }, }, diff --git a/plugins/tech-insights-backend/api-report.md b/plugins/tech-insights-backend/api-report.md index fc1bd9217b..6bd161c092 100644 --- a/plugins/tech-insights-backend/api-report.md +++ b/plugins/tech-insights-backend/api-report.md @@ -15,21 +15,22 @@ import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { TechInsightCheck } from '@backstage/plugin-tech-insights-common'; import { TechInsightsStore } from '@backstage/plugin-tech-insights-common'; +// Warning: (ae-missing-release-tag) "buildTechInsightsContext" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export const buildTechInsightsContext: < + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +>( + options: TechInsightsOptions, +) => Promise>; + // @public export function createRouter< CheckType extends TechInsightCheck, CheckResultType extends CheckResult, >(options: RouterOptions): Promise; -// @public (undocumented) -export class DefaultTechInsightsBuilder< - CheckType extends TechInsightCheck, - CheckResultType extends CheckResult, -> { - constructor(options: TechInsightsOptions); - build(): Promise>; -} - // @public export type PersistenceContext = { techInsightsStore: TechInsightsStore; diff --git a/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js b/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js index 4afe47061f..64f6c11e83 100644 --- a/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js +++ b/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js @@ -24,24 +24,28 @@ exports.up = async function up(knex) { table.comment( 'The table for tech insight fact schemas. Containing a versioned data model definition for a collection of facts.', ); - table.increments('id').primary(); table - .text('ref') + .text('id') .notNullable() .comment('Identifier of the fact retriever plugin/package'); table .string('version') .notNullable() .comment('SemVer string defining the version of schema.'); + table + .string('entityTypes') + .nullable() + .comment( + 'A comma separated collection of entity kinds the fact retriever providing this schema affects. Defaults to null, which means all entity kinds.', + ); table .text('schema') .notNullable() .comment( 'Fact schema defining the values/types what this version of the fact would contain.', ); - - table.index('ref', 'fact_schema_ref_idx'); - table.index(['ref', 'version'], 'fact_schema_ref_version_idx'); + table.primary(['id', 'version']); + table.index('id', 'fact_schema_id_idx'); }); }; @@ -50,8 +54,7 @@ exports.up = async function up(knex) { */ exports.down = async function down(knex) { await knex.schema.alterTable('fact_schemas', table => { - table.dropIndex([], 'fact_schema_ref_idx'); - table.dropIndex([], 'fact_schema_ref_version_idx'); + table.dropIndex([], 'fact_schema_id_idx'); }); await knex.schema.dropTable('fact_schemas'); }; diff --git a/plugins/tech-insights-backend/migrations/202109061212_facts.js b/plugins/tech-insights-backend/migrations/202109061212_facts.js index f80883ef1d..0bdcdc0da1 100644 --- a/plugins/tech-insights-backend/migrations/202109061212_facts.js +++ b/plugins/tech-insights-backend/migrations/202109061212_facts.js @@ -25,11 +25,7 @@ exports.up = async function up(knex) { 'The table for tech insight fact collections. Contains facts for individual fact retriever namespace/ref.', ); table - .bigIncrements('index') - .notNullable() - .comment('An insert counter to ensure ordering'); - table - .text('ref') + .text('id') .notNullable() .comment('Unique identifier of the fact retriever plugin/package'); table @@ -54,9 +50,13 @@ exports.up = async function up(knex) { 'Values of the fact collection stored as key-value pairs in JSON format.', ); - table.index('index', 'fact_index_idx'); - table.index('ref', 'fact_ref_idx'); - table.index(['ref', 'entity'], 'fact_ref_entity_idx'); + table + .foreign(['id', 'version']) + .references(['id', 'version']) + .inTable('fact_schemas'); + + table.index(['id', 'entity'], 'fact_id_entity_idx'); + table.index('id', 'fact_id_idx'); }); }; @@ -65,9 +65,8 @@ exports.up = async function up(knex) { */ exports.down = async function down(knex) { await knex.schema.alterTable('facts', table => { - table.dropIndex([], 'facts_index_idx'); - table.dropIndex([], 'fact_ref_idx'); - table.dropIndex([], 'fact_ref_entity_idx'); + table.dropIndex([], 'fact_id_idx'); + table.dropIndex([], 'fact_id_entity_idx'); }); await knex.schema.dropTable('facts'); }; diff --git a/plugins/tech-insights-backend/src/index.ts b/plugins/tech-insights-backend/src/index.ts index eb54e8cfed..fd7f3ccd53 100644 --- a/plugins/tech-insights-backend/src/index.ts +++ b/plugins/tech-insights-backend/src/index.ts @@ -17,10 +17,10 @@ export * from './service/router'; export type { RouterOptions } from './service/router'; -export { DefaultTechInsightsBuilder } from './service/DefaultTechInsightsBuilder'; +export { buildTechInsightsContext } from './service/techInsightsContextBuilder'; export type { TechInsightsOptions, TechInsightsContext, -} from './service/DefaultTechInsightsBuilder'; +} from './service/techInsightsContextBuilder'; export type { PersistenceContext } from './service/persistence/DatabaseManager'; diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts index 6b9b58e3b8..ef53ae8412 100644 --- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts @@ -16,7 +16,7 @@ import { FactRetriever, FactRetrieverRegistration, - FactSchema, + FactSchemaDefinition, TechInsightFact, TechInsightsStore, } from '@backstage/plugin-tech-insights-common'; @@ -35,21 +35,18 @@ jest.mock('node-cron', () => { }); const testFactRetriever: FactRetriever = { - ref: 'test-factretriever', + id: 'test-factretriever', + version: '0.0.1', + entityTypes: ['component'], schema: { - version: '0.0.1', - schema: { - testnumberfact: { - type: 'integer', - description: '', - entityKinds: ['component'], - }, + testnumberfact: { + type: 'integer', + description: '', }, }, handler: async () => { return [ { - ref: 'test-factretriever', entity: { namespace: 'a', kind: 'a', @@ -65,7 +62,9 @@ const testFactRetriever: FactRetriever = { const cadence = '1 * * * *'; describe('FactRetrieverEngine', () => { let engine: FactRetrieverEngine; - let factSchemaAssertionCallback: (ref: string, schema: FactSchema) => void; + let factSchemaAssertionCallback: ( + factSchemaDefinition: FactSchemaDefinition, + ) => void; let factInsertionAssertionCallback: (facts: TechInsightFact[]) => void; const mockRepository: TechInsightsStore = { @@ -73,8 +72,8 @@ describe('FactRetrieverEngine', () => { factInsertionAssertionCallback(facts); return Promise.resolve(); }, - insertFactSchema: (ref: string, schema: FactSchema) => { - factSchemaAssertionCallback(ref, schema); + insertFactSchema: (def: FactSchemaDefinition) => { + factSchemaAssertionCallback(def); return Promise.resolve(); }, } as unknown as TechInsightsStore; @@ -102,21 +101,19 @@ describe('FactRetrieverEngine', () => { }; it('Should update fact retriever schemas on initialization', async () => { - factSchemaAssertionCallback = (ref, schema) => { - expect(ref).toEqual('test-factretriever'); + factSchemaAssertionCallback = ({ id, schema, version, entityTypes }) => { + expect(id).toEqual('test-factretriever'); + expect(version).toEqual('0.0.1'); + expect(entityTypes).toEqual(['component']); expect(schema).toEqual({ - version: '0.0.1', - schema: { - testnumberfact: { - type: 'integer', - description: '', - entityKinds: ['component'], - }, + testnumberfact: { + type: 'integer', + description: '', }, }); }; - engine = await FactRetrieverEngine.fromConfig(defaultEngineConfig); + engine = await FactRetrieverEngine.create(defaultEngineConfig); }); it('Should insert facts when scheduled step is run', async () => { (schedule as jest.Mock).mockImplementation( @@ -143,7 +140,7 @@ describe('FactRetrieverEngine', () => { }, }); }; - engine = await FactRetrieverEngine.fromConfig(defaultEngineConfig); + engine = await FactRetrieverEngine.create(defaultEngineConfig); engine.schedule(); const job: any = engine.getJob('test-factretriever'); job.triggerScheduledJobNow(); diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts index 01e3c12370..6dc3396819 100644 --- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts @@ -45,7 +45,7 @@ export class FactRetrieverEngine { private readonly defaultCadence?: string, ) {} - static async fromConfig({ + static async create({ repository, factRetrieverRegistry, factRetrieverContext, @@ -59,7 +59,7 @@ export class FactRetrieverEngine { await Promise.all( factRetrieverRegistry .listRetrievers() - .map(it => repository.insertFactSchema(it.ref, it.schema)), + .map(it => repository.insertFactSchema(it)), ); return new FactRetrieverEngine( @@ -76,12 +76,12 @@ export class FactRetrieverEngine { const newRegs: string[] = []; registrations.forEach(registration => { const { factRetriever, cadence } = registration; - if (!this.scheduledJobs.has(factRetriever.ref)) { + if (!this.scheduledJobs.has(factRetriever.id)) { const cronExpression = cadence || this.defaultCadence || randomDailyCron(); if (!validate(cronExpression)) { this.logger.warn( - `Validation failed for cron expression ${cronExpression} when trying to schedule fact retriever ${factRetriever.ref}`, + `Validation failed for cron expression ${cronExpression} when trying to schedule fact retriever ${factRetriever.id}`, ); return; } @@ -89,8 +89,8 @@ export class FactRetrieverEngine { cronExpression, this.createFactRetrieverHandler(factRetriever), ); - this.scheduledJobs.set(factRetriever.ref, job); - newRegs.push(factRetriever.ref); + this.scheduledJobs.set(factRetriever.id, job); + newRegs.push(factRetriever.id); } }); this.logger.info( @@ -106,27 +106,27 @@ export class FactRetrieverEngine { return async () => { const startTimestamp = process.hrtime(); this.logger.info( - `Retrieving facts for fact retriever ${factRetriever.ref}`, + `Retrieving facts for fact retriever ${factRetriever.id}`, ); const facts = await factRetriever.handler(this.factRetrieverContext); if (this.logger.isDebugEnabled()) { this.logger.debug( `Retrieved ${facts.length} facts for fact retriever ${ - factRetriever.ref + factRetriever.id } in ${duration(startTimestamp)}`, ); } try { - await this.repository.insertFacts(factRetriever.ref, facts); + await this.repository.insertFacts(factRetriever.id, facts); this.logger.info( `Stored ${facts.length} facts for fact retriever ${ - factRetriever.ref + factRetriever.id } in ${duration(startTimestamp)}`, ); } catch (e) { this.logger.warn( - `Failed to insert facts for fact retriever ${factRetriever.ref}`, + `Failed to insert facts for fact retriever ${factRetriever.id}`, e, ); } diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts index c76bb6039d..59d1483720 100644 --- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts @@ -31,19 +31,19 @@ export class FactRetrieverRegistry { } register(registration: FactRetrieverRegistration) { - if (this.retrievers.has(registration.factRetriever.ref)) { + if (this.retrievers.has(registration.factRetriever.id)) { throw new ConflictError( - `Tech insight fact retriever with reference '${registration.factRetriever.ref}' has already been registered`, + `Tech insight fact retriever with identifier '${registration.factRetriever.id}' has already been registered`, ); } - this.retrievers.set(registration.factRetriever.ref, registration); + this.retrievers.set(registration.factRetriever.id, registration); } get(retrieverReference: string): FactRetriever { const registration = this.retrievers.get(retrieverReference); if (!registration) { throw new NotFoundError( - `Tech insight fact retriever with reference '${retrieverReference}' is not registered.`, + `Tech insight fact retriever with identifier '${retrieverReference}' is not registered.`, ); } return registration.factRetriever; diff --git a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts index ab54cf819d..44a376d5da 100644 --- a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts +++ b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts @@ -20,47 +20,58 @@ import { Knex } from 'knex'; const factSchemas = [ { - ref: 'test-schema', + id: 'test-fact', version: '0.0.1-test', + entityTypes: ['component'], schema: JSON.stringify({ testNumberFact: { type: 'integer', description: 'Test fact with a number type', - entityKinds: ['component'], }, }), }, ]; const additionalFactSchemas = [ { - ref: 'test-schema', + id: 'test-fact', version: '1.2.1-test', + entityTypes: ['component'], schema: JSON.stringify({ testNumberFact: { type: 'integer', description: 'Test fact with a number type', - entityKinds: ['component'], }, testStringFact: { type: 'string', description: 'Test fact with a string type', - entityKinds: ['service'], }, }), }, { - ref: 'test-schema', + id: 'test-fact', version: '1.1.1-test', + entityTypes: ['component'], schema: JSON.stringify({ testStringFact: { type: 'string', description: 'Test fact with a string type', - entityKinds: ['service'], }, }), }, ]; +const secondSchema = { + id: 'second-test-fact', + version: '0.0.1-test', + entityTypes: ['service'], + schema: JSON.stringify({ + testStringFact: { + type: 'string', + description: 'Test fact with a string type', + }, + }), +}; + const now = DateTime.now().toISO(); const shortlyInTheFuture = DateTime.now() .plus(Duration.fromMillis(555)) @@ -72,18 +83,18 @@ const farInTheFuture = DateTime.now() const facts = [ { timestamp: now, - ref: 'test-fact', + id: 'test-fact', version: '0.0.1-test', - entity: 'a/a/a', + entity: 'a:a/a', facts: JSON.stringify({ testNumberFact: 1, }), }, { timestamp: shortlyInTheFuture, - ref: 'test-fact', + id: 'test-fact', version: '0.0.1-test', - entity: 'a/a/a', + entity: 'a:a/a', facts: JSON.stringify({ testNumberFact: 2, }), @@ -93,9 +104,9 @@ const facts = [ const additionalFacts = [ { timestamp: farInTheFuture, - ref: 'test-fact', + id: 'test-fact', version: '0.0.1-test', - entity: 'a/a/a', + entity: 'a:a/a', facts: JSON.stringify({ testNumberFact: 3, }), @@ -114,7 +125,7 @@ describe('Tech Insights database', () => { }); const baseAssertionFact = { - ref: 'test-fact', + id: 'test-fact', entity: { namespace: 'a', kind: 'a', name: 'a' }, timestamp: DateTime.fromISO(shortlyInTheFuture), version: '0.0.1-test', @@ -124,14 +135,12 @@ describe('Tech Insights database', () => { it('should be able to return latest schema', async () => { const schemas = await store.getLatestSchemas(); expect(schemas[0]).toMatchObject({ - ref: 'test-schema', + id: 'test-fact', version: '0.0.1-test', - schema: { - testNumberFact: { - type: 'integer', - description: 'Test fact with a number type', - entityKinds: ['component'], - }, + entityTypes: ['component'], + testNumberFact: { + type: 'integer', + description: 'Test fact with a number type', }, }); }); @@ -141,43 +150,75 @@ describe('Tech Insights database', () => { const schemas = await store.getLatestSchemas(); expect(schemas[0]).toMatchObject({ - ref: 'test-schema', + id: 'test-fact', version: '1.2.1-test', - schema: { - testNumberFact: { - type: 'integer', - description: 'Test fact with a number type', - entityKinds: ['component'], - }, - testStringFact: { - type: 'string', - description: 'Test fact with a string type', - entityKinds: ['service'], - }, + entityTypes: ['component'], + testNumberFact: { + type: 'integer', + description: 'Test fact with a number type', + }, + testStringFact: { + type: 'string', + description: 'Test fact with a string type', }, }); }); - it('should return latest facts only for the correct ref', async () => { - const returnedFact = await store.getLatestFactsForRefs( + it('should return multiple schemas if those exists', async () => { + await testDbClient.batchInsert('fact_schemas', [ + { + ...secondSchema, + id: 'second', + }, + ]); + + const schemas = await store.getLatestSchemas(); + expect(schemas).toHaveLength(2); + expect(schemas[0]).toMatchObject({ + id: 'test-fact', + version: '1.2.1-test', + entityTypes: ['component'], + testNumberFact: { + type: 'integer', + description: 'Test fact with a number type', + }, + testStringFact: { + type: 'string', + description: 'Test fact with a string type', + }, + }); + expect(schemas[1]).toMatchObject({ + id: 'second', + version: '0.0.1-test', + entityTypes: ['service'], + testStringFact: { + type: 'string', + description: 'Test fact with a string type', + }, + }); + }); + + it('should return latest facts only for the correct id', async () => { + const returnedFact = await store.getLatestFactsByIds( ['test-fact'], - 'a/a/a', + 'a:a/a', ); expect(returnedFact['test-fact']).toMatchObject(baseAssertionFact); }); - it('should return latest facts for multiple refs', async () => { + it('should return latest facts for multiple ids', async () => { + await testDbClient.batchInsert('fact_schemas', [secondSchema]); await testDbClient.batchInsert( 'facts', additionalFacts.map(fact => ({ ...fact, - ref: 'second-test-fact', + id: 'second-test-fact', timestamp: farInTheFuture, })), ); - const returnedFacts = await store.getLatestFactsForRefs( + const returnedFacts = await store.getLatestFactsByIds( ['test-fact', 'second-test-fact'], - 'a/a/a', + 'a:a/a', ); expect(returnedFacts['test-fact']).toMatchObject({ @@ -185,7 +226,7 @@ describe('Tech Insights database', () => { }); expect(returnedFacts['second-test-fact']).toMatchObject({ ...baseAssertionFact, - ref: 'second-test-fact', + id: 'second-test-fact', timestamp: DateTime.fromISO(farInTheFuture), facts: { testNumberFact: 3 }, }); @@ -193,9 +234,9 @@ describe('Tech Insights database', () => { it('should return facts correctly between time range', async () => { await testDbClient.batchInsert('facts', additionalFacts); - const returnedFacts = await store.getFactsBetweenTimestampsForRefs( + const returnedFacts = await store.getFactsBetweenTimestampsByIds( ['test-fact'], - 'a/a/a', + 'a:a/a', DateTime.fromISO(now), DateTime.fromISO(shortlyInTheFuture).plus(Duration.fromMillis(10)), ); diff --git a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts index 1bdf18249f..af45539b32 100644 --- a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts +++ b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts @@ -19,14 +19,16 @@ import { TechInsightFact, FlatTechInsightFact, TechInsightsStore, + FactSchemaDefinition, } from '@backstage/plugin-tech-insights-common'; import { rsort } from 'semver'; -import { groupBy } from 'lodash'; +import { groupBy, omit } from 'lodash'; import { DateTime } from 'luxon'; import { Logger } from 'winston'; +import { parseEntityName, stringifyEntityRef } from '@backstage/catalog-model'; export type RawDbFactRow = { - ref: string; + id: string; version: string; timestamp: Date | string; entity: string; @@ -34,10 +36,10 @@ export type RawDbFactRow = { }; type RawDbFactSchemaRow = { - id: number; - ref: string; + id: string; version: string; schema: string; + entityTypes?: string; }; export class TechInsightsDatabase implements TechInsightsStore { @@ -45,51 +47,51 @@ export class TechInsightsDatabase implements TechInsightsStore { constructor(private readonly db: Knex, private readonly logger: Logger) {} - async getLatestSchemas(refs?: string[]): Promise { + async getLatestSchemas(ids?: string[]): Promise { const queryBuilder = this.db('fact_schemas'); - if (refs) { - queryBuilder.whereIn('ref', refs); + if (ids) { + queryBuilder.whereIn('id', ids); } const existingSchemas = await queryBuilder.orderBy('id', 'desc').select(); - const groupedSchemas = groupBy(existingSchemas, 'ref'); + const groupedSchemas = groupBy(existingSchemas, 'id'); return Object.values(groupedSchemas) .map(schemas => { const sorted = rsort(schemas.map(it => it.version)); return schemas.find(it => it.version === sorted[0])!!; }) .map((it: RawDbFactSchemaRow) => ({ - ...it, - schema: JSON.parse(it.schema), + ...omit(it, 'schema'), + ...JSON.parse(it.schema), + entityTypes: it.entityTypes ? it.entityTypes.split(',') : [], })); } - async insertFactSchema(ref: string, schema: FactSchema) { + async insertFactSchema(schemaDefinition: FactSchemaDefinition) { + const { id, version, schema, entityTypes } = schemaDefinition; const existingSchemas = await this.db('fact_schemas') - .where({ ref }) + .where({ id }) + .and.where({ version }) .select(); - const exists = existingSchemas.some( - it => it.ref === ref && it.version === schema.version, - ); - if (!exists) { + if (!existingSchemas || existingSchemas.length === 0) { await this.db('fact_schemas').insert({ - ref, - version: schema.version, - schema: JSON.stringify(schema.schema), + id, + version, + entityTypes: entityTypes && entityTypes.join(','), + schema: JSON.stringify(schema), }); } } - async insertFacts(ref: string, facts: TechInsightFact[]): Promise { + async insertFacts(id: string, facts: TechInsightFact[]): Promise { if (facts.length === 0) return; - const currentSchema = await this.getLatestSchema(ref); + const currentSchema = await this.getLatestSchema(id); const factRows = facts.map(it => { - const { namespace, name, kind } = it.entity; return { - ref: ref, + id, version: currentSchema.version, - entity: `${namespace}/${kind}/${name}`.toLocaleLowerCase('en-US'), + entity: stringifyEntityRef(it.entity), facts: JSON.stringify(it.facts), ...(it.timestamp && { timestamp: it.timestamp.toJSDate() }), }; @@ -99,36 +101,36 @@ export class TechInsightsDatabase implements TechInsightsStore { }); } - async getLatestFactsForRefs( - refs: string[], + async getLatestFactsByIds( + ids: string[], entityTriplet: string, - ): Promise<{ [p: string]: FlatTechInsightFact }> { + ): Promise<{ [factId: string]: FlatTechInsightFact }> { const results = await this.db('facts') .where({ entity: entityTriplet }) - .and.whereIn('ref', refs) + .and.whereIn('id', ids) .join( this.db('facts') .max('timestamp') - .column('ref as subRef') - .groupBy('ref') + .column('id as subId') + .groupBy('id') .as('subQ'), - 'facts.ref', - 'subQ.subRef', + 'facts.id', + 'subQ.subId', ); return this.dbFactRowsToTechInsightFacts(results); } - async getFactsBetweenTimestampsForRefs( - refs: string[], + async getFactsBetweenTimestampsByIds( + ids: string[], entityTriplet: string, startDateTime: DateTime, endDateTime: DateTime, ): Promise<{ - [p: string]: FlatTechInsightFact[]; + [factId: string]: FlatTechInsightFact[]; }> { const results = await this.db('facts') .where({ entity: entityTriplet }) - .and.whereIn('ref', refs) + .and.whereIn('id', ids) .and.whereBetween('timestamp', [ startDateTime.toISO(), endDateTime.toISO(), @@ -136,31 +138,31 @@ export class TechInsightsDatabase implements TechInsightsStore { return groupBy( results.map(it => { - const [namespace, kind, name] = it.entity.split('/'); + const { namespace, kind, name } = parseEntityName(it.entity); const timestamp = typeof it.timestamp === 'string' ? DateTime.fromISO(it.timestamp) : DateTime.fromJSDate(it.timestamp); return { - ref: it.ref, + id: it.id, entity: { namespace, kind, name }, timestamp, version: it.version, facts: JSON.parse(it.facts), }; }), - 'ref', + 'id', ); } - private async getLatestSchema(ref: string): Promise { + private async getLatestSchema(id: string): Promise { const existingSchemas = await this.db('fact_schemas') - .where({ ref }) + .where({ id }) .orderBy('id', 'desc') .select(); if (existingSchemas.length < 1) { - this.logger.warn(`No schema found for ${ref}. `); - throw new Error(`No schema found for ${ref}. `); + this.logger.warn(`No schema found for ${id}. `); + throw new Error(`No schema found for ${id}. `); } const sorted = rsort(existingSchemas.map(it => it.version)); return existingSchemas.find(it => it.version === sorted[0])!!; @@ -168,15 +170,15 @@ export class TechInsightsDatabase implements TechInsightsStore { private dbFactRowsToTechInsightFacts(rows: RawDbFactRow[]) { return rows.reduce((acc, it) => { - const [namespace, kind, name] = it.entity.split('/'); + const { namespace, kind, name } = parseEntityName(it.entity); const timestamp = typeof it.timestamp === 'string' ? DateTime.fromISO(it.timestamp) : DateTime.fromJSDate(it.timestamp); return { ...acc, - [it.ref]: { - ref: it.ref, + [it.id]: { + id: it.id, entity: { namespace, kind, name }, timestamp, version: it.version, diff --git a/plugins/tech-insights-backend/src/service/router.test.ts b/plugins/tech-insights-backend/src/service/router.test.ts index 1433722e6c..075f67ccfe 100644 --- a/plugins/tech-insights-backend/src/service/router.test.ts +++ b/plugins/tech-insights-backend/src/service/router.test.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { DefaultTechInsightsBuilder } from './DefaultTechInsightsBuilder'; +import { buildTechInsightsContext } from './techInsightsContextBuilder'; import { createRouter } from './router'; import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; @@ -27,14 +27,14 @@ import { Knex } from 'knex'; describe('Tech Insights router tests', () => { let app: express.Express; - const latestFactsForRefsMock = jest.fn(); - const factsBetweenTimestampsForRefsMock = jest.fn(); + const latestFactsByIdsMock = jest.fn(); + const factsBetweenTimestampsByIdsMock = jest.fn(); const latestSchemasMock = jest.fn(); const mockPersistenceContext: PersistenceContext = { techInsightsStore: { - getLatestFactsForRefs: latestFactsForRefsMock, - getFactsBetweenTimestampsForRefs: factsBetweenTimestampsForRefsMock, + getLatestFactsByIds: latestFactsByIdsMock, + getFactsBetweenTimestampsByIds: factsBetweenTimestampsByIdsMock, getLatestSchemas: latestSchemasMock, } as unknown as TechInsightsStore, }; @@ -44,7 +44,7 @@ describe('Tech Insights router tests', () => { }); beforeAll(async () => { - const techInsightsContext = await new DefaultTechInsightsBuilder({ + const techInsightsContext = await buildTechInsightsContext({ database: { getClient: () => { return Promise.resolve({ @@ -61,7 +61,7 @@ describe('Tech Insights router tests', () => { getBaseUrl: (_: string) => Promise.resolve('http://mock.url'), getExternalBaseUrl: (_: string) => Promise.resolve('http://mock.url'), }, - }).build(); + }); const router = await createRouter({ logger: getVoidLogger(), @@ -80,32 +80,36 @@ describe('Tech Insights router tests', () => { it('should not contain check endpoints when checker not present', async () => { await request(app).get('/checks').expect(404); - await request(app).get('/checks/a/a/a').expect(404); + await request(app).post('/checks/a/a/a').expect(404); }); - it('should parse be able to parse ref request params for fact retrieval', async () => { + it('should be able to parse id request params for fact retrieval', async () => { await request(app) - .get('/facts/latest/a/a/a') - .query({ refs: ['firstref', 'secondref'] }) + .get('/facts/latest') + .query({ + entity: 'a:a/a', + ids: ['firstId', 'secondId'], + }) .expect(200); - expect(latestFactsForRefsMock).toHaveBeenCalledWith( - ['firstref', 'secondref'], - 'a/a/a', + expect(latestFactsByIdsMock).toHaveBeenCalledWith( + ['firstId', 'secondId'], + 'a:a/a', ); }); - it('should parse be able to parse datetime request params for fact retrieval', async () => { + it('should be able to parse datetime request params for fact retrieval', async () => { await request(app) - .get('/facts/range/a/a/a') + .get('/facts/range') .query({ - refs: ['firstref', 'secondref'], + entity: 'a:a/a', + ids: ['firstId', 'secondId'], startDatetime: '2021-12-12T12:12:12', endDatetime: '2022-11-11T11:11:11', }) .expect(200); - expect(factsBetweenTimestampsForRefsMock).toHaveBeenCalledWith( - ['firstref', 'secondref'], - 'a/a/a', + expect(factsBetweenTimestampsByIdsMock).toHaveBeenCalledWith( + ['firstId', 'secondId'], + 'a:a/a', DateTime.fromISO('2021-12-12T12:12:12.000+00:00'), DateTime.fromISO('2022-11-11T11:11:11.000+00:00'), ); @@ -113,13 +117,14 @@ describe('Tech Insights router tests', () => { it('should respond gracefully on parsing errors', async () => { await request(app) - .get('/facts/range/a/a/a') + .get('/facts/range') .query({ - refs: ['firstref', 'secondref'], + entity: 'a:a/a', + ids: ['firstId', 'secondId'], startDatetime: '2021-12-1222T12:12:12', endDatetime: '2022-1122-11T11:11:11', }) .expect(422); - expect(latestFactsForRefsMock).toHaveBeenCalledTimes(0); + expect(latestFactsByIdsMock).toHaveBeenCalledTimes(0); }); }); diff --git a/plugins/tech-insights-backend/src/service/router.ts b/plugins/tech-insights-backend/src/service/router.ts index cc41868cd8..d02796117b 100644 --- a/plugins/tech-insights-backend/src/service/router.ts +++ b/plugins/tech-insights-backend/src/service/router.ts @@ -25,6 +25,11 @@ import { import { Logger } from 'winston'; import { DateTime } from 'luxon'; import { PersistenceContext } from './persistence/DatabaseManager'; +import { + EntityRef, + parseEntityName, + stringifyEntityRef, +} from '@backstage/catalog-model'; /** * @public @@ -92,7 +97,7 @@ export async function createRouter< }); } const { checks }: { checks: string[] } = req.body; - const entityTriplet = `${namespace.toLowerCase()}/${kind.toLowerCase()}/${name.toLowerCase()}`; + const entityTriplet = stringifyEntityRef({ namespace, kind, name }); const checkResult = await factChecker.runChecks(entityTriplet, checks); return res.send(checkResult); } catch (e) { @@ -106,22 +111,33 @@ export async function createRouter< } router.get('/fact-schemas', async (req, res) => { - const refs = req.query.refs as string[]; - return res.send(await techInsightsStore.getLatestSchemas(refs)); + const ids = req.query.ids as string[]; + return res.send(await techInsightsStore.getLatestSchemas(ids)); }); - router.get('/facts/latest/:namespace/:kind/:name', async (req, res) => { - const { namespace, kind, name } = req.params; - const refs = req.query.refs as string[]; - const entityTriplet = `${namespace.toLowerCase()}/${kind.toLowerCase()}/${name.toLowerCase()}`; + /** + * /facts/latest?entity=component:default/mycomponent&ids[]=factRetrieverId1&ids[]=factRetrieverId2 + */ + router.get('/facts/latest', async (req, res) => { + const { entity } = req.query; + const { namespace, kind, name } = parseEntityName(entity as EntityRef); + const ids = req.query.ids as string[]; return res.send( - await techInsightsStore.getLatestFactsForRefs(refs, entityTriplet), + await techInsightsStore.getLatestFactsByIds( + ids, + stringifyEntityRef({ namespace, kind, name }), + ), ); }); - router.get('/facts/range/:namespace/:kind/:name', async (req, res) => { - const { namespace, kind, name } = req.params; - const refs = req.query.refs as string[]; + /** + * /facts/latest?entity=component:default/mycomponent&startDateTime=2021-12-24T01:23:45&endDateTime=2021-12-31T23:59:59&ids[]=factRetrieverId1&ids[]=factRetrieverId2 + */ + router.get('/facts/range', async (req, res) => { + const { entity } = req.query; + const { namespace, kind, name } = parseEntityName(entity as EntityRef); + + const ids = req.query.ids as string[]; const startDatetime = DateTime.fromISO(req.query.startDatetime as string); const endDatetime = DateTime.fromISO(req.query.endDatetime as string); if (!startDatetime.isValid || !endDatetime.isValid) { @@ -131,10 +147,10 @@ export async function createRouter< value: !startDatetime.isValid ? startDatetime : endDatetime, }); } - const entityTriplet = `${namespace.toLowerCase()}/${kind.toLowerCase()}/${name.toLowerCase()}`; + const entityTriplet = stringifyEntityRef({ namespace, kind, name }); return res.send( - await techInsightsStore.getFactsBetweenTimestampsForRefs( - refs, + await techInsightsStore.getFactsBetweenTimestampsByIds( + ids, entityTriplet, startDatetime, endDatetime, diff --git a/plugins/tech-insights-backend/src/service/DefaultTechInsightsBuilder.ts b/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts similarity index 63% rename from plugins/tech-insights-backend/src/service/DefaultTechInsightsBuilder.ts rename to plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts index 6c5b946c4c..bf30cdcf38 100644 --- a/plugins/tech-insights-backend/src/service/DefaultTechInsightsBuilder.ts +++ b/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts @@ -80,70 +80,57 @@ export type TechInsightsContext< }; /** - * @public - * @typeParam CheckType - Type of the check for the fact checker this builder returns - * @typeParam CheckResultType - Type of the check result for the fact checker this builder returns + * Constructs needed persistence context, fact retriever engine + * and optionally fact checker implementations to be used in the tech insights module. * - * Default implementation of TechInsightsBuilder. + * @param options - Needed options to construct TechInsightsContext + * @returns TechInsightsContext with persistence implementations and optionally an implementation of a FactChecker */ -export class DefaultTechInsightsBuilder< +export const buildTechInsightsContext = async < CheckType extends TechInsightCheck, CheckResultType extends CheckResult, -> { - private readonly options: TechInsightsOptions; +>( + options: TechInsightsOptions, +): Promise> => { + const { + factRetrievers, + factCheckerFactory, + config, + discovery, + database, + logger, + } = options; - constructor(options: TechInsightsOptions) { - this.options = options; - } + const factRetrieverRegistry = new FactRetrieverRegistry(factRetrievers); - /** - * Constructs needed persistence context, fact retriever engine - * and optionally fact checker implementations to be used in the tech insights module. - * - * @returns TechInsightsContext with persistence implementations and optionally an implementation of a FactChecker - */ - async build(): Promise> { - const { - factRetrievers, - factCheckerFactory, + const persistenceContext = await DatabaseManager.initializePersistenceContext( + await database.getClient(), + { logger }, + ); + + const factRetrieverEngine = await FactRetrieverEngine.create({ + repository: persistenceContext.techInsightsStore, + factRetrieverRegistry, + factRetrieverContext: { config, discovery, - database, logger, - } = this.options; + }, + }); - const factRetrieverRegistry = new FactRetrieverRegistry(factRetrievers); - - const persistenceContext = - await DatabaseManager.initializePersistenceContext( - await database.getClient(), - { logger }, - ); - - const factRetrieverEngine = await FactRetrieverEngine.fromConfig({ - repository: persistenceContext.techInsightsStore, - factRetrieverRegistry, - factRetrieverContext: { - config, - discovery, - logger, - }, - }); - - factRetrieverEngine.schedule(); - - if (factCheckerFactory) { - const factChecker = factCheckerFactory.construct( - persistenceContext.techInsightsStore, - ); - return { - persistenceContext, - factChecker, - }; - } + factRetrieverEngine.schedule(); + if (factCheckerFactory) { + const factChecker = factCheckerFactory.construct( + persistenceContext.techInsightsStore, + ); return { persistenceContext, + factChecker, }; } -} + + return { + persistenceContext, + }; +}; diff --git a/plugins/tech-insights-common/CHANGELOG.md b/plugins/tech-insights-common/CHANGELOG.md deleted file mode 100644 index 17e0211ab0..0000000000 --- a/plugins/tech-insights-common/CHANGELOG.md +++ /dev/null @@ -1,7 +0,0 @@ -# @backstage/plugin-tech-insights-backend - -## 0.0.1 - -### Patch Changes - -- Initial implementation diff --git a/plugins/tech-insights-common/api-report.md b/plugins/tech-insights-common/api-report.md index db987546c3..e83ba18137 100644 --- a/plugins/tech-insights-common/api-report.md +++ b/plugins/tech-insights-common/api-report.md @@ -17,7 +17,7 @@ export interface BooleanCheckResult extends CheckResult { // @public export interface CheckResponse { description: string; - factRefs: string[]; + factIds: string[]; id: string; metadata?: Record; name: string; @@ -68,22 +68,23 @@ export interface FactCheckerFactory< // @public export type FactResponse = { - [key: string]: { - ref: string; + [id: string]: { + id: string; type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; description: string; value: number | string | boolean | DateTime | []; since?: string; metadata?: Record; - entityKinds: string[]; }; }; // @public export interface FactRetriever { + entityTypes?: string[]; handler: (ctx: FactRetrieverContext) => Promise; - ref: string; + id: string; schema: FactSchema; + version: string; } // @public @@ -101,30 +102,28 @@ export type FactRetrieverRegistration = { // @public export type FactSchema = { - version: string; - schema: FactValueDefinitions; -}; - -// @public -export type FactValueDefinitions = { - [key: string]: { + [name: string]: { type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; description: string; since?: string; metadata?: Record; - entityKinds: string[]; }; }; +// Warning: (ae-missing-release-tag) "FactSchemaDefinition" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type FactSchemaDefinition = Omit; + // @public export type FlatTechInsightFact = TechInsightFact & { - ref: string; + id: string; }; // @public export interface TechInsightCheck { description: string; - factRefs: string[]; + factIds: string[]; failureMetadata?: Record; id: string; name: string; @@ -151,14 +150,24 @@ export type TechInsightFact = { kind: string; name: string; }; - facts: Record; + facts: Record< + string, + | number + | string + | boolean + | DateTime + | number[] + | string[] + | boolean[] + | DateTime[] + >; timestamp?: DateTime; }; // @public export interface TechInsightsStore { - getFactsBetweenTimestampsForRefs( - refs: string[], + getFactsBetweenTimestampsByIds( + ids: string[], entity: string, startDateTime: DateTime, endDateTime: DateTime, @@ -166,15 +175,15 @@ export interface TechInsightsStore { [factRef: string]: FlatTechInsightFact[]; }>; // (undocumented) - getLatestFactsForRefs( - refs: string[], + getLatestFactsByIds( + ids: string[], entity: string, ): Promise<{ [factRef: string]: FlatTechInsightFact; }>; - getLatestSchemas(refs?: string[]): Promise; - insertFacts(ref: string, facts: TechInsightFact[]): Promise; - insertFactSchema(ref: string, schema: FactSchema): Promise; + getLatestSchemas(ids?: string[]): Promise; + insertFacts(id: string, facts: TechInsightFact[]): Promise; + insertFactSchema(schemaDefinition: FactSchemaDefinition): Promise; } // (No @packageDocumentation comment for this package) diff --git a/plugins/tech-insights-common/src/checks.ts b/plugins/tech-insights-common/src/checks.ts index 7b2c4c6b8f..90fa7ee34f 100644 --- a/plugins/tech-insights-common/src/checks.ts +++ b/plugins/tech-insights-common/src/checks.ts @@ -157,7 +157,7 @@ export interface TechInsightCheck { * * References the fact container, aka fact retriever itself which may or may not contain multiple individual facts and values */ - factRefs: string[]; + factIds: string[]; /** * Metadata to be returned in case a check has been successfully evaluated diff --git a/plugins/tech-insights-common/src/facts.ts b/plugins/tech-insights-common/src/facts.ts index 1ae507adf8..a46e7bd95d 100644 --- a/plugins/tech-insights-common/src/facts.ts +++ b/plugins/tech-insights-common/src/facts.ts @@ -42,7 +42,17 @@ export type TechInsightFact = { * * Key indicates fact name as it is defined in FactSchema */ - facts: Record; + facts: Record< + string, + | number + | string + | boolean + | DateTime + | number[] + | string[] + | boolean[] + | DateTime[] + >; /** * Optional timestamp value which can be used to override retrieval time of the fact row. @@ -62,7 +72,7 @@ export type FlatTechInsightFact = TechInsightFact & { /** * Reference and unique identifier of the fact row */ - ref: string; + id: string; }; /** @@ -73,8 +83,11 @@ export type FlatTechInsightFact = TechInsightFact & { * Used as part of a schema to validate, identify and generically construct usage implementations * of individual fact values in the system. */ -export type FactValueDefinitions = { - [key: string]: { +export type FactSchema = { + /** + * Name of the fact + */ + [name: string]: { /** * Type of the individual fact value * @@ -109,30 +122,9 @@ export type FactValueDefinitions = { * ``` */ metadata?: Record; - - /** - * A list of entity kind descriptors to indicate if this fact is valid for an entity kind - */ - entityKinds: string[]; }; }; -/** - * @public - * - * Container for FactSchema - */ -export type FactSchema = { - /** - * Semver string indicating the version of this schema - */ - version: string; - /** - * Actual schema definitions for this schema - */ - schema: FactValueDefinitions; -}; - /** * @public * @@ -159,7 +151,15 @@ export interface FactRetriever { * Used to identify and store individual facts returned from this retriever * and schemas defined by this retriever. */ - ref: string; + id: string; + + /** + * Semver string indicating the version of this fact retriever + * This version is used to determine if the schema this fact retriever matches the data this fact retriever provides. + * + * Should be incremented on changes to returned data from the handler or if the schema changes. + */ + version: string; /** * Handler function that needs to be implemented to retrieve fact values for entities. @@ -173,8 +173,20 @@ export interface FactRetriever { * A fact schema defining the shape of data returned from the handler method for each entity */ schema: FactSchema; + + /** + * An optional list of entity type descriptors to indicate if this fact retriever is valid for an entity type. + * If omitted, the retriever should apply to all entities. + * + * Should be defined as: + * ['component', 'group', 'user'] for top level items + * ['component:service', 'component:website'] for component types. + */ + entityTypes?: string[]; } +export type FactSchemaDefinition = Omit; + /** * @public * diff --git a/plugins/tech-insights-common/src/persistence.ts b/plugins/tech-insights-common/src/persistence.ts index 9ffb6a9a28..444e961008 100644 --- a/plugins/tech-insights-common/src/persistence.ts +++ b/plugins/tech-insights-common/src/persistence.ts @@ -13,7 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { FactSchema, TechInsightFact, FlatTechInsightFact } from './facts'; +import { + FactSchema, + TechInsightFact, + FlatTechInsightFact, + FactSchemaDefinition, +} from './facts'; import { DateTime } from 'luxon'; /** @@ -28,34 +33,34 @@ export interface TechInsightsStore { * * Each row may contain multiple individual facts and values * - * @param ref - Unique identifier of the fact retriever these facts relate to + * @param id - Unique identifier of the fact retriever these facts relate to * @param facts - A collection of TechInsightFacts */ - insertFacts(ref: string, facts: TechInsightFact[]): Promise; + insertFacts(id: string, facts: TechInsightFact[]): Promise; /** - * @param refs - A collection of reference string to a fact row + * @param ids - A collection of fact row identifiers * @param entity - A string identifying an entity. In a format namespace/kind/name * * @returns - An object keyed by a fact reference and containing an individual TechInsightFact */ - getLatestFactsForRefs( - refs: string[], + getLatestFactsByIds( + ids: string[], entity: string, ): Promise<{ [factRef: string]: FlatTechInsightFact }>; /** * Retrieves fact values identified by fact row references for an individual entity. * - * @param refs - A collection of reference string to a fact row + * @param ids - A collection of fact row identifiers * @param entity - A string identifying an entity. In a format namespace/kind/name * @param startDateTime - DateTime object indicating start of the time frame * @param endDateTime - DateTime object indicating start of the time frame * * @returns - An object keyed by a fact reference and containing a collection of TechInsightFacts matching the time frame */ - getFactsBetweenTimestampsForRefs( - refs: string[], + getFactsBetweenTimestampsByIds( + ids: string[], entity: string, startDateTime: DateTime, endDateTime: DateTime, @@ -64,16 +69,15 @@ export interface TechInsightsStore { /** * Stores versioned fact schemas into data store * - * @param ref - Identifier of the fact schema. Reference to a fact retriever. - * @param schema - The actual schema to store + * @param schemaDefinition - FactSchemaDefinition containing id, version, schema and entityTypes. */ - insertFactSchema(ref: string, schema: FactSchema): Promise; + insertFactSchema(schemaDefinition: FactSchemaDefinition): Promise; /** * Retrieves latest versions (as defined by semver) of fact schemas from the data store. * - * @param refs - Collection of refs to return. If omitted, all Schemas should be returned. + * @param ids - Collection of ids to return. If omitted, all Schemas should be returned. * @returns - A collection of schemas */ - getLatestSchemas(refs?: string[]): Promise; + getLatestSchemas(ids?: string[]): Promise; } diff --git a/plugins/tech-insights-common/src/responses.ts b/plugins/tech-insights-common/src/responses.ts index 0cd1d0a2e8..c3dee1df48 100644 --- a/plugins/tech-insights-common/src/responses.ts +++ b/plugins/tech-insights-common/src/responses.ts @@ -44,7 +44,7 @@ export interface CheckResponse { /** * A collection of references to fact rows used to run this checks against */ - factRefs: string[]; + factIds: string[]; /** * Metadata related to a check. @@ -62,11 +62,11 @@ export interface CheckResponse { * Keyed by the name of the fact */ export type FactResponse = { - [key: string]: { + [id: string]: { /** * Reference and unique identifier of the fact row */ - ref: string; + id: string; /** * Type of the individual fact value * @@ -97,10 +97,5 @@ export type FactResponse = { * Currently loosely typed, but in the future when patterns emerge, key shapes can be defined */ metadata?: Record; - - /** - * A list of entity kind descriptors to indicate if this fact is valid for an entity kind - */ - entityKinds: string[]; }; }; From 2b3e959ef47e37b2e944356e4c26bb12ae451e97 Mon Sep 17 00:00:00 2001 From: Jussi Hallila Date: Thu, 28 Oct 2021 16:20:44 +0200 Subject: [PATCH 32/58] Modifications after PR review * Change entity filter to be an actual entity filter instead of a list of kinds or types * Modify/simplify types a little bit * Split common type libs to one for node and one isomorphic * Remove unnecessary items from FactChecker interface to simplify execution loop. Needs still matching README.md changes. Signed-off-by: Jussi Hallila --- packages/backend/package.json | 2 +- packages/backend/src/plugins/techInsights.ts | 72 +++++---- .../api-report.md | 16 +- .../package.json | 6 +- .../src/service/CheckRegistry.ts | 2 +- .../JsonRulesEngineFactChecker.test.ts | 2 +- .../src/service/JsonRulesEngineFactChecker.ts | 42 ++--- .../src/types.ts | 2 +- plugins/tech-insights-backend/README.md | 39 +++-- plugins/tech-insights-backend/api-report.md | 19 ++- .../migrations/202109061111_fact_schemas.js | 4 +- plugins/tech-insights-backend/package.json | 1 + plugins/tech-insights-backend/src/index.ts | 1 + .../service/fact/FactRetrieverEngine.test.ts | 8 +- .../src/service/fact/FactRetrieverEngine.ts | 2 +- .../src/service/fact/FactRetrieverRegistry.ts | 2 +- .../src/service/fact/createFactRetriever.ts | 49 ++++++ .../service/persistence/DatabaseManager.ts | 2 +- .../persistence/TechInsightsDatabase.test.ts | 18 +-- .../persistence/TechInsightsDatabase.ts | 10 +- .../src/service/router.test.ts | 2 +- .../src/service/router.ts | 5 +- .../src/service/techInsightsContextBuilder.ts | 6 +- plugins/tech-insights-common/README.md | 2 +- plugins/tech-insights-common/api-report.md | 147 ------------------ plugins/tech-insights-common/package.json | 5 +- plugins/tech-insights-common/src/index.ts | 112 ++++++++++++- plugins/tech-insights-common/src/responses.ts | 101 ------------ plugins/tech-insights-node/.eslintrc.js | 3 + plugins/tech-insights-node/README.md | 3 + plugins/tech-insights-node/package.json | 46 ++++++ .../src/checks.ts | 54 +------ .../src/facts.ts | 12 +- plugins/tech-insights-node/src/index.test.ts | 24 +++ plugins/tech-insights-node/src/index.ts | 19 +++ .../src/persistence.ts | 0 .../src/setupTests.ts | 0 37 files changed, 393 insertions(+), 447 deletions(-) create mode 100644 plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts delete mode 100644 plugins/tech-insights-common/src/responses.ts create mode 100644 plugins/tech-insights-node/.eslintrc.js create mode 100644 plugins/tech-insights-node/README.md create mode 100644 plugins/tech-insights-node/package.json rename plugins/{tech-insights-common => tech-insights-node}/src/checks.ts (77%) rename plugins/{tech-insights-common => tech-insights-node}/src/facts.ts (92%) create mode 100644 plugins/tech-insights-node/src/index.test.ts create mode 100644 plugins/tech-insights-node/src/index.ts rename plugins/{tech-insights-common => tech-insights-node}/src/persistence.ts (100%) rename plugins/{tech-insights-common => tech-insights-node}/src/setupTests.ts (100%) diff --git a/packages/backend/package.json b/packages/backend/package.json index a56b302954..f1d58252a4 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -49,7 +49,7 @@ "@backstage/plugin-search-backend-module-pg": "^0.2.1", "@backstage/plugin-techdocs-backend": "^0.10.5", "@backstage/plugin-tech-insights-backend": "^0.1.0", - "@backstage/plugin-tech-insights-common": "^0.1.0", + "@backstage/plugin-tech-insights-node": "^0.1.0", "@backstage/plugin-tech-insights-backend-module-jsonfc": "^0.1.0", "@backstage/plugin-todo-backend": "^0.1.13", "@gitbeaker/node": "^30.2.0", diff --git a/packages/backend/src/plugins/techInsights.ts b/packages/backend/src/plugins/techInsights.ts index d51c0c7575..9927d98f87 100644 --- a/packages/backend/src/plugins/techInsights.ts +++ b/packages/backend/src/plugins/techInsights.ts @@ -16,14 +16,15 @@ import { createRouter, buildTechInsightsContext, + createFactRetrieverRegistration, } from '@backstage/plugin-tech-insights-backend'; import { Router } from 'express'; import { PluginEnvironment } from '../types'; -import { - JSON_RULE_ENGINE_CHECK_TYPE, - JsonRulesEngineFactCheckerFactory, -} from '@backstage/plugin-tech-insights-backend-module-jsonfc'; import { CatalogClient } from '@backstage/catalog-client'; +import { + JsonRulesEngineFactCheckerFactory, + JSON_RULE_ENGINE_CHECK_TYPE, +} from '@backstage/plugin-tech-insights-backend-module-jsonfc'; export default async function createPlugin({ logger, @@ -37,41 +38,38 @@ export default async function createPlugin({ database, discovery, factRetrievers: [ - { - cadence: '1 1 1 * *', // At 01:01 on day-of-month 1. - factRetriever: { - id: 'testRetriever', - version: '1.1.1', - entityTypes: ['component'], - schema: { - examplenumberfact: { - type: 'integer', - description: '', - }, - }, - handler: async _ctx => { - const catalogClient = new CatalogClient({ - discoveryApi: discovery, - }); - const entities = await catalogClient.getEntities(); - - return Promise.resolve( - entities.items.map(it => { - return { - entity: { - namespace: it.metadata.namespace!!, - kind: it.kind, - name: it.metadata.name, - }, - facts: { - examplenumberfact: 2, - }, - }; - }), - ); + createFactRetrieverRegistration('* * * * *', { + id: 'testRetriever', + version: '1.1.2', + entityFilter: [{ kind: 'component' }], // EntityFilter to be used in the future (creating checks, graphs etc.) to figure out which entities this fact retrieves data for. + schema: { + examplenumberfact: { + type: 'integer', + description: 'Example fact returning a number', }, }, - }, + handler: async _ctx => { + const catalogClient = new CatalogClient({ + discoveryApi: discovery, + }); + const entities = await catalogClient.getEntities(); + + return Promise.resolve( + entities.items.map(it => { + return { + entity: { + namespace: it.metadata.namespace!!, + kind: it.kind, + name: it.metadata.name, + }, + facts: { + examplenumberfact: 2, + }, + }; + }), + ); + }, + }), ], factCheckerFactory: new JsonRulesEngineFactCheckerFactory({ checks: [ diff --git a/plugins/tech-insights-backend-module-jsonfc/api-report.md b/plugins/tech-insights-backend-module-jsonfc/api-report.md index 3d214566c5..7093fbfc2b 100644 --- a/plugins/tech-insights-backend-module-jsonfc/api-report.md +++ b/plugins/tech-insights-backend-module-jsonfc/api-report.md @@ -5,12 +5,12 @@ ```ts import { BooleanCheckResult } from '@backstage/plugin-tech-insights-common'; import { CheckResponse } from '@backstage/plugin-tech-insights-common'; -import { CheckValidationResponse } from '@backstage/plugin-tech-insights-common'; -import { FactChecker } from '@backstage/plugin-tech-insights-common'; +import { CheckValidationResponse } from '@backstage/plugin-tech-insights-node'; +import { FactChecker } from '@backstage/plugin-tech-insights-node'; import { Logger as Logger_2 } from 'winston'; -import { TechInsightCheck } from '@backstage/plugin-tech-insights-common'; -import { TechInsightCheckRegistry } from '@backstage/plugin-tech-insights-common'; -import { TechInsightsStore } from '@backstage/plugin-tech-insights-common'; +import { TechInsightCheck } from '@backstage/plugin-tech-insights-node'; +import { TechInsightCheckRegistry } from '@backstage/plugin-tech-insights-node'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; import { TopLevelCondition } from 'json-rules-engine'; // @public (undocumented) @@ -52,13 +52,11 @@ export class JsonRulesEngineFactChecker checkRegistry, }: JsonRulesEngineFactCheckerOptions); // (undocumented) - addCheck(check: TechInsightJsonRuleCheck): Promise; - // (undocumented) getChecks(): Promise; // (undocumented) runChecks( entity: string, - checks: string[], + checks?: string[], ): Promise; // (undocumented) validate(check: TechInsightJsonRuleCheck): Promise; @@ -79,7 +77,7 @@ export class JsonRulesEngineFactCheckerFactory { export type JsonRulesEngineFactCheckerFactoryOptions = { checks: TechInsightJsonRuleCheck[]; logger: Logger_2; - checkRegistry?: TechInsightCheckRegistry; + checkRegistry?: TechInsightCheckRegistry; }; // @public diff --git a/plugins/tech-insights-backend-module-jsonfc/package.json b/plugins/tech-insights-backend-module-jsonfc/package.json index c11f20ee38..a1db34a97e 100644 --- a/plugins/tech-insights-backend-module-jsonfc/package.json +++ b/plugins/tech-insights-backend-module-jsonfc/package.json @@ -1,8 +1,8 @@ { "name": "@backstage/plugin-tech-insights-backend-module-jsonfc", "version": "0.1.0", - "main": "dist/index.cjs.js", - "types": "dist/index.d.ts", + "main": "src/index.ts", + "types": "src/index.ts", "license": "Apache-2.0", "private": false, "publishConfig": { @@ -35,11 +35,11 @@ "@backstage/config": "^0.1.8", "@backstage/errors": "^0.1.1", "@backstage/plugin-tech-insights-common": "^0.1.0", + "@backstage/plugin-tech-insights-node": "^0.1.0", "ajv": "^7.0.3", "json-rules-engine": "^6.1.2", "lodash": "^4.17.21", "luxon": "^2.0.2", - "node-cron": "^3.0.0", "winston": "^3.2.1" }, "devDependencies": { diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts index 016fb05b31..dec84a8634 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts @@ -18,7 +18,7 @@ import { ConflictError, NotFoundError } from '@backstage/errors'; import { TechInsightCheck, TechInsightCheckRegistry, -} from '@backstage/plugin-tech-insights-common'; +} from '@backstage/plugin-tech-insights-node'; export class DefaultCheckRegistry implements TechInsightCheckRegistry diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts index a31ae90c70..77b3e6c8b1 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts @@ -17,7 +17,7 @@ import { TechInsightCheckRegistry, TechInsightsStore, -} from '@backstage/plugin-tech-insights-common'; +} from '@backstage/plugin-tech-insights-node'; import { JSON_RULE_ENGINE_CHECK_TYPE, JsonRulesEngineFactCheckerFactory, diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts index ab90a3d828..ae1f18a373 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts @@ -17,13 +17,12 @@ import { JsonRuleBooleanCheckResult, TechInsightJsonRuleCheck } from '../types'; import { FactChecker, - FactResponse, TechInsightCheckRegistry, FlatTechInsightFact, TechInsightsStore, CheckValidationResponse, - CheckValidationError, -} from '@backstage/plugin-tech-insights-common'; +} from '@backstage/plugin-tech-insights-node'; +import { FactResponse } from '@backstage/plugin-tech-insights-common'; import { Engine, EngineResult, TopLevelCondition } from 'json-rules-engine'; import { DefaultCheckRegistry } from './CheckRegistry'; import { Logger } from 'winston'; @@ -71,15 +70,19 @@ export class JsonRulesEngineFactChecker this.repository = repository; this.logger = logger; checks.forEach(check => this.validate(check)); - this.checkRegistry = checkRegistry ?? new DefaultCheckRegistry(checks); + this.checkRegistry = + checkRegistry ?? + new DefaultCheckRegistry(checks); } async runChecks( entity: string, - checks: string[], + checks?: string[], ): Promise { const engine = new Engine(); - const techInsightChecks = await this.checkRegistry.getAll(checks); + const techInsightChecks = checks + ? await this.checkRegistry.getAll(checks) + : await this.checkRegistry.list(); const factIds = techInsightChecks.flatMap(it => it.factIds); const facts = await this.repository.getLatestFactsByIds(factIds, entity); techInsightChecks.forEach(techInsightCheck => { @@ -127,7 +130,7 @@ export class JsonRulesEngineFactChecker return { valid: false, message: msg, - errors: validator.errors, + errors: validator.errors ? validator.errors : undefined, }; } @@ -166,21 +169,6 @@ export class JsonRulesEngineFactChecker return this.checkRegistry.list(); } - async addCheck( - check: TechInsightJsonRuleCheck, - ): Promise { - const checkValidationResponse = await this.validate(check); - if (!checkValidationResponse.valid) { - const msg = `Check validation failed when adding check ${check.name} to check registry.`; - this.logger.warn(msg); - throw new CheckValidationError({ - message: checkValidationResponse.message || msg, - errors: checkValidationResponse.errors, - }); - } - return await this.checkRegistry.register(check); - } - private retrieveIndividualFactReferences( condition: TopLevelCondition | { fact: string }, ): string[] { @@ -255,8 +243,10 @@ export class JsonRulesEngineFactChecker }; if ('toJSON' in result) { - // Results serialize "wrong" since the objects are creating their own serialization implementations - // 'toJSON' should always be present in the result object but it is missing from the types + // Results from json-rules-engine serialize "wrong" since the objects are creating their own serialization implementations. + // 'toJSON' should always be present in the result object but it is missing from the types. + // Parsing the stringified representation into a plain object here to be able to serialize it later + // along with other items present in the returned response. const rule = JSON.parse(result.toJSON()); return { ...returnable, rule: pick(rule, ['conditions']) }; } @@ -312,7 +302,7 @@ export class JsonRulesEngineFactChecker export type JsonRulesEngineFactCheckerFactoryOptions = { checks: TechInsightJsonRuleCheck[]; logger: Logger; - checkRegistry?: TechInsightCheckRegistry; + checkRegistry?: TechInsightCheckRegistry; }; /** @@ -325,7 +315,7 @@ export type JsonRulesEngineFactCheckerFactoryOptions = { export class JsonRulesEngineFactCheckerFactory { private readonly checks: TechInsightJsonRuleCheck[]; private readonly logger: Logger; - private readonly checkRegistry?: TechInsightCheckRegistry; + private readonly checkRegistry?: TechInsightCheckRegistry; constructor({ checks, diff --git a/plugins/tech-insights-backend-module-jsonfc/src/types.ts b/plugins/tech-insights-backend-module-jsonfc/src/types.ts index 23d5849eda..592fee248d 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/types.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/types.ts @@ -14,10 +14,10 @@ * limitations under the License. */ import { TopLevelCondition } from 'json-rules-engine'; +import { TechInsightCheck } from '@backstage/plugin-tech-insights-node'; import { BooleanCheckResult, CheckResponse, - TechInsightCheck, } from '@backstage/plugin-tech-insights-common'; /** diff --git a/plugins/tech-insights-backend/README.md b/plugins/tech-insights-backend/README.md index 32265c1ced..2de16b65b0 100644 --- a/plugins/tech-insights-backend/README.md +++ b/plugins/tech-insights-backend/README.md @@ -77,16 +77,18 @@ you will not have any fact retrievers present in your application. To have the i To create factRetrieverRegistration you need to implement `FactRetriever` interface defined in `@backstage/plugin-tech-insights-common` package. After you have implemented this interface you can wrap that into a registration object like follows: ```ts -const myFactRetriever: FactRetriever = { +import { createFactRetrieverRegistration } from './createFactRetriever'; + +const myFactRetriever = { /** * snip */ }; -const myFactRetrieverRegistration = { - cadence: '1 * 3 * * ', // On the first minute of the third day of the month - factRetriever: myFactRetriever, -}; +const myFactRetrieverRegistration = createFactRetrieverRegistration( + '1 * 3 * * ', // On the first minute of the third day of the month + myFactRetriever, +); ``` Then you can modify the example `techInsights.ts` file shown above like this: @@ -104,28 +106,25 @@ discovery, ### Creating Fact Retrievers -A Fact Retriever consist of three parts: +A Fact Retriever consist of four parts: -1. `ref` - unique identifier of a fact retriever -2. `schema` - A versioned schema defining the shape of data a fact retriever returns -3. `handler` - An asynchronous function handling the logic of retrieving and returning facts for an entity +1. `id` - unique identifier of a fact retriever +2. `version`: A semver string indicating the current version of the schema and the handler +3. `schema` - A versioned schema defining the shape of data a fact retriever returns +4. `handler` - An asynchronous function handling the logic of retrieving and returning facts for an entity An example implementation of a FactRetriever could for example be as follows: ```ts const myFactRetriever: FactRetriever = { - ref: 'documentation-number-factretriever', // unique ref, identifier of the fact retriever + id: 'documentation-number-factretriever', // unique identifier of the fact retriever + version: '0.1.1', // SemVer version number of this fact retriever schema. This should be incremented if the implementation changes schema: { - version: '0.1.1', // SemVer version number of this fact retriever schema. This should be incremented if the implementation changes - - // the actual schema - schema: { - // Name/identifier of an individual fact that this retriever returns - examplenumberfact: { - type: 'integer', // Type of the fact - description: 'A fact of a number', // Description of the fact - entityTypes: ['component'], // An array of entity kinds that this fact is applicable to - }, + // Name/identifier of an individual fact that this retriever returns + examplenumberfact: { + type: 'integer', // Type of the fact + description: 'A fact of a number', // Description of the fact + entityTypes: ['component'], // An array of entity kinds that this fact is applicable to }, }, handler: async ctx => { diff --git a/plugins/tech-insights-backend/api-report.md b/plugins/tech-insights-backend/api-report.md index 6bd161c092..826789dd1d 100644 --- a/plugins/tech-insights-backend/api-report.md +++ b/plugins/tech-insights-backend/api-report.md @@ -6,17 +6,16 @@ import { CheckResult } from '@backstage/plugin-tech-insights-common'; import { Config } from '@backstage/config'; import express from 'express'; -import { FactChecker } from '@backstage/plugin-tech-insights-common'; -import { FactCheckerFactory } from '@backstage/plugin-tech-insights-common'; -import { FactRetrieverRegistration } from '@backstage/plugin-tech-insights-common'; +import { FactChecker } from '@backstage/plugin-tech-insights-node'; +import { FactCheckerFactory } from '@backstage/plugin-tech-insights-node'; +import { FactRetriever } from '@backstage/plugin-tech-insights-node'; +import { FactRetrieverRegistration } from '@backstage/plugin-tech-insights-node'; import { Logger as Logger_2 } from 'winston'; import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; -import { TechInsightCheck } from '@backstage/plugin-tech-insights-common'; -import { TechInsightsStore } from '@backstage/plugin-tech-insights-common'; +import { TechInsightCheck } from '@backstage/plugin-tech-insights-node'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; -// Warning: (ae-missing-release-tag) "buildTechInsightsContext" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public export const buildTechInsightsContext: < CheckType extends TechInsightCheck, @@ -25,6 +24,12 @@ export const buildTechInsightsContext: < options: TechInsightsOptions, ) => Promise>; +// @public +export function createFactRetrieverRegistration( + cadence: string, + factRetriever: FactRetriever, +): FactRetrieverRegistration; + // @public export function createRouter< CheckType extends TechInsightCheck, diff --git a/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js b/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js index 64f6c11e83..6094daa4b7 100644 --- a/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js +++ b/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js @@ -33,10 +33,10 @@ exports.up = async function up(knex) { .notNullable() .comment('SemVer string defining the version of schema.'); table - .string('entityTypes') + .string('entityFilter') .nullable() .comment( - 'A comma separated collection of entity kinds the fact retriever providing this schema affects. Defaults to null, which means all entity kinds.', + 'A serialized entity filter object used to determine which entities this schema is applicable to.', ); table .text('schema') diff --git a/plugins/tech-insights-backend/package.json b/plugins/tech-insights-backend/package.json index fba5daae45..cf1e1d1c6f 100644 --- a/plugins/tech-insights-backend/package.json +++ b/plugins/tech-insights-backend/package.json @@ -37,6 +37,7 @@ "@backstage/config": "^0.1.8", "@backstage/errors": "^0.1.1", "@backstage/plugin-tech-insights-common": "^0.1.0", + "@backstage/plugin-tech-insights-node": "^0.1.0", "@types/express": "^4.17.6", "cross-fetch": "^3.0.6", "express": "^4.17.1", diff --git a/plugins/tech-insights-backend/src/index.ts b/plugins/tech-insights-backend/src/index.ts index fd7f3ccd53..2a74bc6b88 100644 --- a/plugins/tech-insights-backend/src/index.ts +++ b/plugins/tech-insights-backend/src/index.ts @@ -24,3 +24,4 @@ export type { } from './service/techInsightsContextBuilder'; export type { PersistenceContext } from './service/persistence/DatabaseManager'; +export { createFactRetrieverRegistration } from './service/fact/createFactRetriever'; diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts index ef53ae8412..4dc002465b 100644 --- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts @@ -19,7 +19,7 @@ import { FactSchemaDefinition, TechInsightFact, TechInsightsStore, -} from '@backstage/plugin-tech-insights-common'; +} from '@backstage/plugin-tech-insights-node'; import { FactRetrieverRegistry } from './FactRetrieverRegistry'; import { FactRetrieverEngine } from './FactRetrieverEngine'; import { getVoidLogger } from '@backstage/backend-common'; @@ -37,7 +37,7 @@ jest.mock('node-cron', () => { const testFactRetriever: FactRetriever = { id: 'test-factretriever', version: '0.0.1', - entityTypes: ['component'], + entityFilter: [{ kind: 'component' }], schema: { testnumberfact: { type: 'integer', @@ -101,10 +101,10 @@ describe('FactRetrieverEngine', () => { }; it('Should update fact retriever schemas on initialization', async () => { - factSchemaAssertionCallback = ({ id, schema, version, entityTypes }) => { + factSchemaAssertionCallback = ({ id, schema, version, entityFilter }) => { expect(id).toEqual('test-factretriever'); expect(version).toEqual('0.0.1'); - expect(entityTypes).toEqual(['component']); + expect(entityFilter).toEqual([{ kind: 'component' }]); expect(schema).toEqual({ testnumberfact: { type: 'integer', diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts index 6dc3396819..aa2207cca8 100644 --- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts @@ -17,7 +17,7 @@ import { FactRetriever, FactRetrieverContext, TechInsightsStore, -} from '@backstage/plugin-tech-insights-common'; +} from '@backstage/plugin-tech-insights-node'; import { FactRetrieverRegistry } from './FactRetrieverRegistry'; import { schedule, validate, ScheduledTask } from 'node-cron'; import { Logger } from 'winston'; diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts index 59d1483720..719f8bf504 100644 --- a/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts @@ -18,7 +18,7 @@ import { FactRetriever, FactRetrieverRegistration, FactSchema, -} from '@backstage/plugin-tech-insights-common'; +} from '@backstage/plugin-tech-insights-node'; import { ConflictError, NotFoundError } from '@backstage/errors'; export class FactRetrieverRegistry { diff --git a/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts b/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts new file mode 100644 index 0000000000..b744d3b826 --- /dev/null +++ b/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts @@ -0,0 +1,49 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + FactRetriever, + FactRetrieverRegistration, +} from '@backstage/plugin-tech-insights-node'; + +/** + * @public + * + * A helper function to construct fact retriever registrations. + * + * Cron expressions help: + * ┌────────────── second (optional) + # │ ┌──────────── minute + # │ │ ┌────────── hour + # │ │ │ ┌──────── day of month + # │ │ │ │ ┌────── month + # │ │ │ │ │ ┌──── day of week + # │ │ │ │ │ │ + # │ │ │ │ │ │ + # * * * * * * + * + * + * @param cadence - cron expression to indicate when the fact retriever should be triggered + * @param factRetriever - Implementation of fact retriever consisting of at least id, version, schema and handler + */ +export function createFactRetrieverRegistration( + cadence: string, + factRetriever: FactRetriever, +): FactRetrieverRegistration { + return { + cadence, + factRetriever, + }; +} diff --git a/plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts b/plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts index 75fb313f80..126c253fa5 100644 --- a/plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts +++ b/plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts @@ -18,7 +18,7 @@ import knexFactory, { Knex } from 'knex'; import { Logger } from 'winston'; import { v4 as uuidv4 } from 'uuid'; import { TechInsightsDatabase } from './TechInsightsDatabase'; -import { TechInsightsStore } from '@backstage/plugin-tech-insights-common'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; const migrationsDir = resolvePackagePath( '@backstage/plugin-tech-insights-backend', diff --git a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts index 44a376d5da..85d6098ce3 100644 --- a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts +++ b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts @@ -15,14 +15,14 @@ */ import { DatabaseManager } from './DatabaseManager'; import { DateTime, Duration } from 'luxon'; -import { TechInsightsStore } from '@backstage/plugin-tech-insights-common'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; import { Knex } from 'knex'; const factSchemas = [ { id: 'test-fact', version: '0.0.1-test', - entityTypes: ['component'], + entityFilter: JSON.stringify([{ kind: 'component' }]), schema: JSON.stringify({ testNumberFact: { type: 'integer', @@ -35,7 +35,7 @@ const additionalFactSchemas = [ { id: 'test-fact', version: '1.2.1-test', - entityTypes: ['component'], + entityFilter: JSON.stringify([{ kind: 'component' }]), schema: JSON.stringify({ testNumberFact: { type: 'integer', @@ -50,7 +50,7 @@ const additionalFactSchemas = [ { id: 'test-fact', version: '1.1.1-test', - entityTypes: ['component'], + entityFilter: JSON.stringify([{ kind: 'component' }]), schema: JSON.stringify({ testStringFact: { type: 'string', @@ -63,7 +63,7 @@ const additionalFactSchemas = [ const secondSchema = { id: 'second-test-fact', version: '0.0.1-test', - entityTypes: ['service'], + entityFilter: JSON.stringify([{ kind: 'service' }]), schema: JSON.stringify({ testStringFact: { type: 'string', @@ -137,7 +137,7 @@ describe('Tech Insights database', () => { expect(schemas[0]).toMatchObject({ id: 'test-fact', version: '0.0.1-test', - entityTypes: ['component'], + entityFilter: [{ kind: 'component' }], testNumberFact: { type: 'integer', description: 'Test fact with a number type', @@ -152,7 +152,7 @@ describe('Tech Insights database', () => { expect(schemas[0]).toMatchObject({ id: 'test-fact', version: '1.2.1-test', - entityTypes: ['component'], + entityFilter: [{ kind: 'component' }], testNumberFact: { type: 'integer', description: 'Test fact with a number type', @@ -177,7 +177,7 @@ describe('Tech Insights database', () => { expect(schemas[0]).toMatchObject({ id: 'test-fact', version: '1.2.1-test', - entityTypes: ['component'], + entityFilter: [{ kind: 'component' }], testNumberFact: { type: 'integer', description: 'Test fact with a number type', @@ -190,7 +190,7 @@ describe('Tech Insights database', () => { expect(schemas[1]).toMatchObject({ id: 'second', version: '0.0.1-test', - entityTypes: ['service'], + entityFilter: [{ kind: 'service' }], testStringFact: { type: 'string', description: 'Test fact with a string type', diff --git a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts index af45539b32..31a2997af5 100644 --- a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts +++ b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts @@ -20,7 +20,7 @@ import { FlatTechInsightFact, TechInsightsStore, FactSchemaDefinition, -} from '@backstage/plugin-tech-insights-common'; +} from '@backstage/plugin-tech-insights-node'; import { rsort } from 'semver'; import { groupBy, omit } from 'lodash'; import { DateTime } from 'luxon'; @@ -39,7 +39,7 @@ type RawDbFactSchemaRow = { id: string; version: string; schema: string; - entityTypes?: string; + entityFilter?: string; }; export class TechInsightsDatabase implements TechInsightsStore { @@ -63,12 +63,12 @@ export class TechInsightsDatabase implements TechInsightsStore { .map((it: RawDbFactSchemaRow) => ({ ...omit(it, 'schema'), ...JSON.parse(it.schema), - entityTypes: it.entityTypes ? it.entityTypes.split(',') : [], + entityFilter: it.entityFilter ? JSON.parse(it.entityFilter) : null, })); } async insertFactSchema(schemaDefinition: FactSchemaDefinition) { - const { id, version, schema, entityTypes } = schemaDefinition; + const { id, version, schema, entityFilter } = schemaDefinition; const existingSchemas = await this.db('fact_schemas') .where({ id }) .and.where({ version }) @@ -78,7 +78,7 @@ export class TechInsightsDatabase implements TechInsightsStore { await this.db('fact_schemas').insert({ id, version, - entityTypes: entityTypes && entityTypes.join(','), + entityFilter: entityFilter ? JSON.stringify(entityFilter) : undefined, schema: JSON.stringify(schema), }); } diff --git a/plugins/tech-insights-backend/src/service/router.test.ts b/plugins/tech-insights-backend/src/service/router.test.ts index 075f67ccfe..8a7b6bdbb3 100644 --- a/plugins/tech-insights-backend/src/service/router.test.ts +++ b/plugins/tech-insights-backend/src/service/router.test.ts @@ -20,7 +20,7 @@ import { ConfigReader } from '@backstage/config'; import request from 'supertest'; import express from 'express'; import { PersistenceContext } from './persistence/DatabaseManager'; -import { TechInsightsStore } from '@backstage/plugin-tech-insights-common'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; import { DateTime } from 'luxon'; import { Knex } from 'knex'; diff --git a/plugins/tech-insights-backend/src/service/router.ts b/plugins/tech-insights-backend/src/service/router.ts index d02796117b..752534928f 100644 --- a/plugins/tech-insights-backend/src/service/router.ts +++ b/plugins/tech-insights-backend/src/service/router.ts @@ -20,8 +20,9 @@ import { Config } from '@backstage/config'; import { FactChecker, TechInsightCheck, - CheckResult, -} from '@backstage/plugin-tech-insights-common'; +} from '@backstage/plugin-tech-insights-node'; + +import { CheckResult } from '@backstage/plugin-tech-insights-common'; import { Logger } from 'winston'; import { DateTime } from 'luxon'; import { PersistenceContext } from './persistence/DatabaseManager'; diff --git a/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts b/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts index bf30cdcf38..7305f1778c 100644 --- a/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts +++ b/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts @@ -23,16 +23,16 @@ import { PluginEndpointDiscovery, } from '@backstage/backend-common'; import { - CheckResult, FactChecker, FactCheckerFactory, FactRetrieverRegistration, TechInsightCheck, -} from '@backstage/plugin-tech-insights-common'; +} from '@backstage/plugin-tech-insights-node'; import { DatabaseManager, PersistenceContext, } from './persistence/DatabaseManager'; +import { CheckResult } from '@backstage/plugin-tech-insights-common'; /** * @public @@ -80,6 +80,8 @@ export type TechInsightsContext< }; /** + * @public + * * Constructs needed persistence context, fact retriever engine * and optionally fact checker implementations to be used in the tech insights module. * diff --git a/plugins/tech-insights-common/README.md b/plugins/tech-insights-common/README.md index 651c0e969d..af14ef0e0d 100644 --- a/plugins/tech-insights-common/README.md +++ b/plugins/tech-insights-common/README.md @@ -1,3 +1,3 @@ # Tech Insights Common -Common types and functionalities for tech insights, to be shared between tech-insights-backend and individual implementations of FactRetrievers, FactCheckers and their persistence options. +Common types and functionalities for tech insights shared in an isomorphic manner between BE and FE implementations. diff --git a/plugins/tech-insights-common/api-report.md b/plugins/tech-insights-common/api-report.md index e83ba18137..ddb38cfa85 100644 --- a/plugins/tech-insights-common/api-report.md +++ b/plugins/tech-insights-common/api-report.md @@ -3,10 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { Config } from '@backstage/config'; import { DateTime } from 'luxon'; -import { Logger as Logger_2 } from 'winston'; -import { PluginEndpointDiscovery } from '@backstage/backend-common'; // @public export interface BooleanCheckResult extends CheckResult { @@ -30,42 +27,6 @@ export type CheckResult = { check: CheckResponse; }; -// @public -export class CheckValidationError extends Error { - constructor({ message, errors }: { message: string; errors?: any }); - // (undocumented) - errors?: any; -} - -// @public -export type CheckValidationResponse = { - valid: boolean; - message?: string; - errors?: any; -}; - -// @public -export interface FactChecker< - CheckType extends TechInsightCheck, - CheckResultType extends CheckResult, -> { - addCheck(check: CheckType): Promise; - getChecks(): Promise; - runChecks(entity: string, checks: string[]): Promise; - validate(check: CheckType): Promise; -} - -// @public -export interface FactCheckerFactory< - CheckType extends TechInsightCheck, - CheckResultType extends CheckResult, -> { - // (undocumented) - construct( - repository: TechInsightsStore, - ): FactChecker; -} - // @public export type FactResponse = { [id: string]: { @@ -78,113 +39,5 @@ export type FactResponse = { }; }; -// @public -export interface FactRetriever { - entityTypes?: string[]; - handler: (ctx: FactRetrieverContext) => Promise; - id: string; - schema: FactSchema; - version: string; -} - -// @public -export type FactRetrieverContext = { - config: Config; - discovery: PluginEndpointDiscovery; - logger: Logger_2; -}; - -// @public -export type FactRetrieverRegistration = { - factRetriever: FactRetriever; - cadence?: string; -}; - -// @public -export type FactSchema = { - [name: string]: { - type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; - description: string; - since?: string; - metadata?: Record; - }; -}; - -// Warning: (ae-missing-release-tag) "FactSchemaDefinition" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export type FactSchemaDefinition = Omit; - -// @public -export type FlatTechInsightFact = TechInsightFact & { - id: string; -}; - -// @public -export interface TechInsightCheck { - description: string; - factIds: string[]; - failureMetadata?: Record; - id: string; - name: string; - successMetadata?: Record; - type: string; -} - -// @public -export interface TechInsightCheckRegistry { - // (undocumented) - get(checkId: string): Promise; - // (undocumented) - getAll(checks: string[]): Promise; - // (undocumented) - list(): Promise; - // (undocumented) - register(check: CheckType): Promise; -} - -// @public -export type TechInsightFact = { - entity: { - namespace: string; - kind: string; - name: string; - }; - facts: Record< - string, - | number - | string - | boolean - | DateTime - | number[] - | string[] - | boolean[] - | DateTime[] - >; - timestamp?: DateTime; -}; - -// @public -export interface TechInsightsStore { - getFactsBetweenTimestampsByIds( - ids: string[], - entity: string, - startDateTime: DateTime, - endDateTime: DateTime, - ): Promise<{ - [factRef: string]: FlatTechInsightFact[]; - }>; - // (undocumented) - getLatestFactsByIds( - ids: string[], - entity: string, - ): Promise<{ - [factRef: string]: FlatTechInsightFact; - }>; - getLatestSchemas(ids?: string[]): Promise; - insertFacts(id: string, facts: TechInsightFact[]): Promise; - insertFactSchema(schemaDefinition: FactSchemaDefinition): Promise; -} - // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/tech-insights-common/package.json b/plugins/tech-insights-common/package.json index 37d66a9379..4c287c26f3 100644 --- a/plugins/tech-insights-common/package.json +++ b/plugins/tech-insights-common/package.json @@ -30,11 +30,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.9.0", - "@backstage/config": "^0.1.8", "@types/luxon": "^2.0.5", - "luxon": "^2.0.2", - "winston": "^3.2.1" + "luxon": "^2.0.2" }, "devDependencies": { "@backstage/cli": "^0.7.1" diff --git a/plugins/tech-insights-common/src/index.ts b/plugins/tech-insights-common/src/index.ts index 982ae0ac41..4bfc660748 100644 --- a/plugins/tech-insights-common/src/index.ts +++ b/plugins/tech-insights-common/src/index.ts @@ -14,7 +14,111 @@ * limitations under the License. */ -export * from './checks'; -export * from './facts'; -export * from './persistence'; -export * from './responses'; +import { DateTime } from 'luxon'; + +/** + * @public + * + * Response type for checks. + */ +export interface CheckResponse { + /** + * Identifier of the Check + */ + id: string; + /** + * Type identifier for the check. + * Can be used to determine storage options, logical routing to correct FactChecker implementation + * or to help frontend render correct component types based on this + */ + type: string; + /** + * Human readable name of the Check + */ + name: string; + /** + * Description of the Check + */ + description: string; + + /** + * A collection of references to fact rows used to run this checks against + */ + factIds: string[]; + + /** + * Metadata related to a check. + * Can contain links, additional description texts and other actionable data. + * + * Currently loosely typed, but in the future when patterns emerge, key shapes can be defined + */ + metadata?: Record; +} + +/** + * @public + * + * Individual fact response type. + * Keyed by the name of the fact + */ +export type FactResponse = { + [id: string]: { + /** + * Reference and unique identifier of the fact row + */ + id: string; + /** + * Type of the individual fact value + * + * Numbers are split into integers and floating point values. + * `set` indicates a collection of values + */ + type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; + + /** + * Description of the individual fact + */ + description: string; + + /** + * Actual value of the fact + */ + value: number | string | boolean | DateTime | []; + + /** + * An optional SemVer version identifying when this fact was added to the FactSchema + */ + since?: string; + + /** + * Metadata related to an individual fact. + * Can contain links, additional description texts and other actionable data. + * + * Currently loosely typed, but in the future when patterns emerge, key shapes can be defined + */ + metadata?: Record; + }; +}; + +/** + * Generic CheckResult + * + * Contains information about the facts used to calculate the check result + * and information about the check itself. Both may include metadata to be able to display additional information. + * A collection of these should be parseable by the frontend to display scorecards + * + * @public + */ +export type CheckResult = { + facts: FactResponse; + check: CheckResponse; +}; + +/** + * CheckResult of type Boolean. + * + * @public + */ +export interface BooleanCheckResult extends CheckResult { + result: boolean; +} diff --git a/plugins/tech-insights-common/src/responses.ts b/plugins/tech-insights-common/src/responses.ts deleted file mode 100644 index c3dee1df48..0000000000 --- a/plugins/tech-insights-common/src/responses.ts +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { DateTime } from 'luxon'; - -/** - * @public - * - * Response type for checks. - */ -export interface CheckResponse { - /** - * Identifier of the Check - */ - id: string; - /** - * Type identifier for the check. - * Can be used to determine storage options, logical routing to correct FactChecker implementation - * or to help frontend render correct component types based on this - */ - type: string; - /** - * Human readable name of the Check - */ - name: string; - /** - * Description of the Check - */ - description: string; - - /** - * A collection of references to fact rows used to run this checks against - */ - factIds: string[]; - - /** - * Metadata related to a check. - * Can contain links, additional description texts and other actionable data. - * - * Currently loosely typed, but in the future when patterns emerge, key shapes can be defined - */ - metadata?: Record; -} - -/** - * @public - * - * Individual fact response type. - * Keyed by the name of the fact - */ -export type FactResponse = { - [id: string]: { - /** - * Reference and unique identifier of the fact row - */ - id: string; - /** - * Type of the individual fact value - * - * Numbers are split into integers and floating point values. - * `set` indicates a collection of values - */ - type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; - - /** - * Description of the individual fact - */ - description: string; - - /** - * Actual value of the fact - */ - value: number | string | boolean | DateTime | []; - - /** - * An optional SemVer version identifying when this fact was added to the FactSchema - */ - since?: string; - - /** - * Metadata related to an individual fact. - * Can contain links, additional description texts and other actionable data. - * - * Currently loosely typed, but in the future when patterns emerge, key shapes can be defined - */ - metadata?: Record; - }; -}; diff --git a/plugins/tech-insights-node/.eslintrc.js b/plugins/tech-insights-node/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/tech-insights-node/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/tech-insights-node/README.md b/plugins/tech-insights-node/README.md new file mode 100644 index 0000000000..4065611837 --- /dev/null +++ b/plugins/tech-insights-node/README.md @@ -0,0 +1,3 @@ +# Tech Insights Node + +Common types and functionalities for tech insights backend implementations to be shared between tech-insights-backend and individual implementations of FactRetrievers, FactCheckers and their persistence options. diff --git a/plugins/tech-insights-node/package.json b/plugins/tech-insights-node/package.json new file mode 100644 index 0000000000..e5620322fd --- /dev/null +++ b/plugins/tech-insights-node/package.json @@ -0,0 +1,46 @@ +{ + "name": "@backstage/plugin-tech-insights-node", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/tech-insights-node" + }, + "keywords": [ + "backstage", + "tech-insights" + ], + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/backend-common": "^0.9.0", + "@backstage/config": "^0.1.8", + "@backstage/plugin-tech-insights-common": "^0.1.0 ", + "@types/luxon": "^2.0.5", + "luxon": "^2.0.2", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/cli": "^0.7.1" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/tech-insights-common/src/checks.ts b/plugins/tech-insights-node/src/checks.ts similarity index 77% rename from plugins/tech-insights-common/src/checks.ts rename to plugins/tech-insights-node/src/checks.ts index 90fa7ee34f..e5f10aff6d 100644 --- a/plugins/tech-insights-common/src/checks.ts +++ b/plugins/tech-insights-node/src/checks.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import { TechInsightsStore } from './persistence'; -import { CheckResponse, FactResponse } from './responses'; +import { CheckResult } from '@backstage/plugin-tech-insights-common'; /** * A factory wrapper to construct FactChecker implementations. @@ -57,16 +57,7 @@ export interface FactChecker< * @param checks - A collection of checks to run against provided entity * @returns - A collection containing check/fact information and the actual results of the check */ - runChecks(entity: string, checks: string[]): Promise; - - /** - * Adds and stores new checks so they can be run checks against. - * Implementation should ideally run validation against the check. - * - * @param check - The actual check to be added. - * @returns - An indicator if fact was successfully added - */ - addCheck(check: CheckType): Promise; + runChecks(entity: string, checks?: string[]): Promise; /** * Retrieves all available checks that can be used to run checks against. @@ -99,29 +90,6 @@ export interface TechInsightCheckRegistry { list(): Promise; } -/** - * Generic CheckResult - * - * Contains information about the facts used to calculate the check result - * and information about the check itself. Both may include metadata to be able to display additional information. - * A collection of these should be parseable by the frontend to display scorecards - * - * @public - */ -export type CheckResult = { - facts: FactResponse; - check: CheckResponse; -}; - -/** - * CheckResult of type Boolean. - * - * @public - */ -export interface BooleanCheckResult extends CheckResult { - result: boolean; -} - /** * Generic definition of a check for Tech Insights * @@ -181,21 +149,5 @@ export interface TechInsightCheck { export type CheckValidationResponse = { valid: boolean; message?: string; - errors?: any; + errors?: unknown[]; }; - -/** - * Error object for Validation Errors - * - * Can be used to short circuit larger execution paths instead of passing validation response around - * - * @public - */ -export class CheckValidationError extends Error { - errors?: any; - - constructor({ message, errors }: { message: string; errors?: any }) { - super(message); - this.errors = errors; - } -} diff --git a/plugins/tech-insights-common/src/facts.ts b/plugins/tech-insights-node/src/facts.ts similarity index 92% rename from plugins/tech-insights-common/src/facts.ts rename to plugins/tech-insights-node/src/facts.ts index a46e7bd95d..b912895ec6 100644 --- a/plugins/tech-insights-common/src/facts.ts +++ b/plugins/tech-insights-node/src/facts.ts @@ -175,14 +175,16 @@ export interface FactRetriever { schema: FactSchema; /** - * An optional list of entity type descriptors to indicate if this fact retriever is valid for an entity type. + * An optional object/array of objects of entity filters to indicate if this fact retriever is valid for an entity type. * If omitted, the retriever should apply to all entities. * - * Should be defined as: - * ['component', 'group', 'user'] for top level items - * ['component:service', 'component:website'] for component types. + * Should be defined for example: + * { field: 'kind', values: ['component'] } + * { field: 'metadata.name', values: ['component-1', 'component-2'] } */ - entityTypes?: string[]; + entityFilter?: + | Record[] + | Record; } export type FactSchemaDefinition = Omit; diff --git a/plugins/tech-insights-node/src/index.test.ts b/plugins/tech-insights-node/src/index.test.ts new file mode 100644 index 0000000000..5e5f8c9d77 --- /dev/null +++ b/plugins/tech-insights-node/src/index.test.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as anything from './'; + +describe('tech-insights-node', () => { + // Types only + it('should exist', () => { + expect(anything).toBeTruthy(); + }); +}); diff --git a/plugins/tech-insights-node/src/index.ts b/plugins/tech-insights-node/src/index.ts new file mode 100644 index 0000000000..70ef27e159 --- /dev/null +++ b/plugins/tech-insights-node/src/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './checks'; +export * from './facts'; +export * from './persistence'; diff --git a/plugins/tech-insights-common/src/persistence.ts b/plugins/tech-insights-node/src/persistence.ts similarity index 100% rename from plugins/tech-insights-common/src/persistence.ts rename to plugins/tech-insights-node/src/persistence.ts diff --git a/plugins/tech-insights-common/src/setupTests.ts b/plugins/tech-insights-node/src/setupTests.ts similarity index 100% rename from plugins/tech-insights-common/src/setupTests.ts rename to plugins/tech-insights-node/src/setupTests.ts From da4577d73183d641607e87a7102335aea8d6f680 Mon Sep 17 00:00:00 2001 From: Jussi Hallila Date: Thu, 28 Oct 2021 16:25:53 +0200 Subject: [PATCH 33/58] Un-yikesify example FactRetriever cron to not pollute the DB Signed-off-by: Jussi Hallila --- packages/backend/src/plugins/techInsights.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/plugins/techInsights.ts b/packages/backend/src/plugins/techInsights.ts index 9927d98f87..6e10b60370 100644 --- a/packages/backend/src/plugins/techInsights.ts +++ b/packages/backend/src/plugins/techInsights.ts @@ -38,7 +38,7 @@ export default async function createPlugin({ database, discovery, factRetrievers: [ - createFactRetrieverRegistration('* * * * *', { + createFactRetrieverRegistration('5 4 * * 6', { id: 'testRetriever', version: '1.1.2', entityFilter: [{ kind: 'component' }], // EntityFilter to be used in the future (creating checks, graphs etc.) to figure out which entities this fact retrieves data for. From 29faa2ace2149abbc782ce5f3c2f80db2eb87d99 Mon Sep 17 00:00:00 2001 From: Jussi Hallila Date: Fri, 29 Oct 2021 10:09:34 +0200 Subject: [PATCH 34/58] Update cli version Signed-off-by: Jussi Hallila --- plugins/tech-insights-backend-module-jsonfc/package.json | 2 +- plugins/tech-insights-backend/package.json | 2 +- plugins/tech-insights-common/package.json | 2 +- plugins/tech-insights-node/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/tech-insights-backend-module-jsonfc/package.json b/plugins/tech-insights-backend-module-jsonfc/package.json index a1db34a97e..8943b0e461 100644 --- a/plugins/tech-insights-backend-module-jsonfc/package.json +++ b/plugins/tech-insights-backend-module-jsonfc/package.json @@ -43,7 +43,7 @@ "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.7.1", + "@backstage/cli": "^0.8.0", "@types/node-cron": "^2.0.4" }, "files": [ diff --git a/plugins/tech-insights-backend/package.json b/plugins/tech-insights-backend/package.json index cf1e1d1c6f..cd437ee16b 100644 --- a/plugins/tech-insights-backend/package.json +++ b/plugins/tech-insights-backend/package.json @@ -52,7 +52,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.7.1", + "@backstage/cli": "^0.8.0", "@types/supertest": "^2.0.8", "@types/node-cron": "^3.0.0", "@types/semver": "^7.3.8", diff --git a/plugins/tech-insights-common/package.json b/plugins/tech-insights-common/package.json index 4c287c26f3..5e02c9e4b0 100644 --- a/plugins/tech-insights-common/package.json +++ b/plugins/tech-insights-common/package.json @@ -34,7 +34,7 @@ "luxon": "^2.0.2" }, "devDependencies": { - "@backstage/cli": "^0.7.1" + "@backstage/cli": "^0.8.0" }, "files": [ "dist" diff --git a/plugins/tech-insights-node/package.json b/plugins/tech-insights-node/package.json index e5620322fd..4a7cf3589b 100644 --- a/plugins/tech-insights-node/package.json +++ b/plugins/tech-insights-node/package.json @@ -38,7 +38,7 @@ "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.7.1" + "@backstage/cli": "^0.8.0" }, "files": [ "dist" From 5ebfedd9c2bf8f1acc6f3613b27abef9a331c8fd Mon Sep 17 00:00:00 2001 From: Jussi Hallila Date: Fri, 29 Oct 2021 12:29:27 +0200 Subject: [PATCH 35/58] Add new api-report Update persistence context to not be wrapped into a useless class. Modify the way test databases are created. Signed-off-by: Jussi Hallila --- packages/backend/src/plugins/techInsights.ts | 5 +- .../README.md | 30 ++-- plugins/tech-insights-backend/README.md | 52 +++--- plugins/tech-insights-backend/package.json | 1 + plugins/tech-insights-backend/src/index.ts | 2 +- .../service/persistence/DatabaseManager.ts | 99 ----------- .../persistence/TechInsightsDatabase.test.ts | 15 +- .../service/persistence/persistenceContext.ts | 59 +++++++ .../src/service/router.test.ts | 2 +- .../src/service/router.ts | 2 +- .../src/service/techInsightsContextBuilder.ts | 6 +- plugins/tech-insights-node/api-report.md | 155 ++++++++++++++++++ 12 files changed, 278 insertions(+), 150 deletions(-) delete mode 100644 plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts create mode 100644 plugins/tech-insights-backend/src/service/persistence/persistenceContext.ts create mode 100644 plugins/tech-insights-node/api-report.md diff --git a/packages/backend/src/plugins/techInsights.ts b/packages/backend/src/plugins/techInsights.ts index 6e10b60370..abab1d133d 100644 --- a/packages/backend/src/plugins/techInsights.ts +++ b/packages/backend/src/plugins/techInsights.ts @@ -39,6 +39,7 @@ export default async function createPlugin({ discovery, factRetrievers: [ createFactRetrieverRegistration('5 4 * * 6', { + // Example cron, At 04:05 on Saturday. id: 'testRetriever', version: '1.1.2', entityFilter: [{ kind: 'component' }], // EntityFilter to be used in the future (creating checks, graphs etc.) to figure out which entities this fact retrieves data for. @@ -52,7 +53,9 @@ export default async function createPlugin({ const catalogClient = new CatalogClient({ discoveryApi: discovery, }); - const entities = await catalogClient.getEntities(); + const entities = await catalogClient.getEntities({ + filter: [{ kind: 'component' }], + }); return Promise.resolve( entities.items.map(it => { diff --git a/plugins/tech-insights-backend-module-jsonfc/README.md b/plugins/tech-insights-backend-module-jsonfc/README.md index 1404bd4ea4..a19c3dd2ce 100644 --- a/plugins/tech-insights-backend-module-jsonfc/README.md +++ b/plugins/tech-insights-backend-module-jsonfc/README.md @@ -24,31 +24,31 @@ and modify the `techInsights.ts` file to contain a reference to the FactCheckers + logger, + }), -const builder = new DefaultTechInsightsBuilder({ -logger, -config, -database, -discovery, -factRetrievers: [myFactRetrieverRegistration], -+ factCheckerFactory: myFactCheckerFactory -}); + const builder = new DefaultTechInsightsBuilder({ + logger, + config, + database, + discovery, + factRetrievers: [myFactRetrieverRegistration], ++ factCheckerFactory: myFactCheckerFactory + }); ``` By default this implementation comes with an in-memory storage to store checks. You can inject an additional data store by adding an implementation of `TechInsightCheckRegistry` into the constructor options when creating a `JsonRulesEngineFactCheckerFactory`. That can be done as follows ```diff -const myTechInsightCheckRegistry: TechInsightCheckRegistry = // snip -const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ - checks: [], - logger, -+ checkRegistry: myTechInsightCheckRegistry -}), + const myTechInsightCheckRegistry: TechInsightCheckRegistry = // snip + const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ + checks: [], + logger, ++ checkRegistry: myTechInsightCheckRegistry + }), ``` ## Adding checks -Checks for this FactChecker are constructed as `json-rules-engine` compatible JSON rules. A check could look like the following for example: +Checks for this FactChecker are constructed as [`json-rules-engine` compatible JSON rules](https://github.com/CacheControl/json-rules-engine/blob/master/docs/rules.md#conditions). A check could look like the following for example: ```ts import { TechInsightJsonRuleCheck } from '../types'; diff --git a/plugins/tech-insights-backend/README.md b/plugins/tech-insights-backend/README.md index 2de16b65b0..e7a78d34a2 100644 --- a/plugins/tech-insights-backend/README.md +++ b/plugins/tech-insights-backend/README.md @@ -53,20 +53,20 @@ With the `techInsights.ts` router setup in place, add the router to `packages/backend/src/index.ts`: ```diff -+import techInsights from './plugins/techInsights'; ++ import techInsights from './plugins/techInsights'; -async function main() { - ... - const createEnv = makeCreateEnv(config); + async function main() { + ... + const createEnv = makeCreateEnv(config); - const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); -+ const techInsightsEnv = useHotMemoize(module, () => createEnv('tech_insights')); - - const apiRouter = Router(); -+ apiRouter.use('/tech-insights', await techInsights(techInsightsEnv)); - ... - apiRouter.use(notFoundHandler()); + const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); ++ const techInsightsEnv = useHotMemoize(module, () => createEnv('tech_insights')); + const apiRouter = Router(); ++ apiRouter.use('/tech-insights', await techInsights(techInsightsEnv)); + ... + apiRouter.use(notFoundHandler()); + } ``` ### Adding fact retrievers @@ -95,10 +95,10 @@ Then you can modify the example `techInsights.ts` file shown above like this: ```diff const builder = new DefaultTechInsightsBuilder({ -logger, -config, -database, -discovery, + logger, + config, + database, + discovery, - factRetrievers: [], + factRetrievers: [myFactRetrieverRegistration], }); @@ -119,12 +119,12 @@ An example implementation of a FactRetriever could for example be as follows: const myFactRetriever: FactRetriever = { id: 'documentation-number-factretriever', // unique identifier of the fact retriever version: '0.1.1', // SemVer version number of this fact retriever schema. This should be incremented if the implementation changes + entityFilter: [{ kind: 'component' }], // EntityFilter to be used in the future (creating checks, graphs etc.) to figure out which entities this fact retrieves data for. schema: { // Name/identifier of an individual fact that this retriever returns examplenumberfact: { type: 'integer', // Type of the fact description: 'A fact of a number', // Description of the fact - entityTypes: ['component'], // An array of entity kinds that this fact is applicable to }, }, handler: async ctx => { @@ -133,7 +133,9 @@ const myFactRetriever: FactRetriever = { const catalogClient = new CatalogClient({ discoveryApi: discovery, }); - const entities = await catalogClient.getEntities(); // Retrieve all entities + const entities = await catalogClient.getEntities({ + filter: [{ kind: 'component' }], + }); /** * snip: Do complex logic to retrieve facts from external system or calculate fact values */ @@ -183,14 +185,14 @@ and modify the `techInsights.ts` file to contain a reference to the FactChecker + logger, + }), -const builder = new DefaultTechInsightsBuilder({ -logger, -config, -database, -discovery, -factRetrievers: [myFactRetrieverRegistration], -+ factCheckerFactory: myFactCheckerFactory -}); + const builder = new DefaultTechInsightsBuilder({ + logger, + config, + database, + discovery, + factRetrievers: [myFactRetrieverRegistration], ++ factCheckerFactory: myFactCheckerFactory + }); ``` To be able to run checks, you need to additionally add individual checks into your FactChecker implementation. For examples how to add these, you can check the documentation of the individual implementation of the FactChecker diff --git a/plugins/tech-insights-backend/package.json b/plugins/tech-insights-backend/package.json index cd437ee16b..678bec25c4 100644 --- a/plugins/tech-insights-backend/package.json +++ b/plugins/tech-insights-backend/package.json @@ -52,6 +52,7 @@ "yn": "^4.0.0" }, "devDependencies": { + "@backstage/backend-test-utils": "^0.1.8", "@backstage/cli": "^0.8.0", "@types/supertest": "^2.0.8", "@types/node-cron": "^3.0.0", diff --git a/plugins/tech-insights-backend/src/index.ts b/plugins/tech-insights-backend/src/index.ts index 2a74bc6b88..5e8e690e8d 100644 --- a/plugins/tech-insights-backend/src/index.ts +++ b/plugins/tech-insights-backend/src/index.ts @@ -23,5 +23,5 @@ export type { TechInsightsContext, } from './service/techInsightsContextBuilder'; -export type { PersistenceContext } from './service/persistence/DatabaseManager'; +export type { PersistenceContext } from './service/persistence/persistenceContext'; export { createFactRetrieverRegistration } from './service/fact/createFactRetriever'; diff --git a/plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts b/plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts deleted file mode 100644 index 126c253fa5..0000000000 --- a/plugins/tech-insights-backend/src/service/persistence/DatabaseManager.ts +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common'; -import knexFactory, { Knex } from 'knex'; -import { Logger } from 'winston'; -import { v4 as uuidv4 } from 'uuid'; -import { TechInsightsDatabase } from './TechInsightsDatabase'; -import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; - -const migrationsDir = resolvePackagePath( - '@backstage/plugin-tech-insights-backend', - 'migrations', -); - -/** - * A Container for persistence related components in TechInsights - * - * @public - */ -export type PersistenceContext = { - techInsightsStore: TechInsightsStore; -}; - -export type CreateDatabaseOptions = { - logger: Logger; -}; - -const defaultOptions: CreateDatabaseOptions = { - logger: getVoidLogger(), -}; - -/** - * A factory class to construct persistence context for both running implmentation and test cases. - * - * @public - */ -export class DatabaseManager { - public static async initializePersistenceContext( - knex: Knex, - options: CreateDatabaseOptions = defaultOptions, - ): Promise { - await knex.migrate.latest({ - directory: migrationsDir, - }); - return { - techInsightsStore: new TechInsightsDatabase(knex, options.logger), - }; - } - - public static async createTestDatabase( - knex: Knex, - ): Promise { - const knexInstance = knex ?? (await this.createTestDatabaseConnection()); - return await this.initializePersistenceContext(knexInstance); - } - - public static async createTestDatabaseConnection(): Promise { - const config: Knex.Config = { - client: 'sqlite3', - connection: ':memory:', - useNullAsDefault: true, - }; - - let knexInstance = knexFactory(config); - if (typeof config.connection !== 'string') { - const tempDbName = `d${uuidv4().replace(/-/g, '')}`; - await knexInstance.raw(`CREATE DATABASE ${tempDbName};`); - knexInstance = knexFactory({ - ...config, - connection: { - ...config.connection, - database: tempDbName, - }, - }); - } - - knexInstance.client.pool.on( - 'createSuccess', - (_eventId: any, resource: any) => { - resource.run('PRAGMA foreign_keys = ON', () => {}); - }, - ); - - return knexInstance; - } -} diff --git a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts index 85d6098ce3..43d00e2425 100644 --- a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts +++ b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts @@ -13,10 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { DatabaseManager } from './DatabaseManager'; import { DateTime, Duration } from 'luxon'; import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; import { Knex } from 'knex'; +import { TestDatabases } from '@backstage/backend-test-utils'; +import { getVoidLogger } from '@backstage/backend-common'; +import { initializePersistenceContext } from './persistenceContext'; const factSchemas = [ { @@ -117,9 +119,14 @@ describe('Tech Insights database', () => { let store: TechInsightsStore; let testDbClient: Knex; beforeAll(async () => { - testDbClient = await DatabaseManager.createTestDatabaseConnection(); - store = (await DatabaseManager.createTestDatabase(testDbClient)) - .techInsightsStore; + testDbClient = await TestDatabases.create().init('SQLITE_3'); + + store = ( + await initializePersistenceContext(testDbClient, { + logger: getVoidLogger(), + }) + ).techInsightsStore; + await testDbClient.batchInsert('fact_schemas', factSchemas); await testDbClient.batchInsert('facts', facts); }); diff --git a/plugins/tech-insights-backend/src/service/persistence/persistenceContext.ts b/plugins/tech-insights-backend/src/service/persistence/persistenceContext.ts new file mode 100644 index 0000000000..b0fb65349f --- /dev/null +++ b/plugins/tech-insights-backend/src/service/persistence/persistenceContext.ts @@ -0,0 +1,59 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common'; +import { Knex } from 'knex'; +import { Logger } from 'winston'; +import { TechInsightsDatabase } from './TechInsightsDatabase'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-tech-insights-backend', + 'migrations', +); + +/** + * A Container for persistence related components in TechInsights + * + * @public + */ +export type PersistenceContext = { + techInsightsStore: TechInsightsStore; +}; + +export type CreateDatabaseOptions = { + logger: Logger; +}; + +const defaultOptions: CreateDatabaseOptions = { + logger: getVoidLogger(), +}; + +/** + * A factory method to construct persistence context for running implementation. + * + * @public + */ +export const initializePersistenceContext = async ( + knex: Knex, + options: CreateDatabaseOptions = defaultOptions, +): Promise => { + await knex.migrate.latest({ + directory: migrationsDir, + }); + return { + techInsightsStore: new TechInsightsDatabase(knex, options.logger), + }; +}; diff --git a/plugins/tech-insights-backend/src/service/router.test.ts b/plugins/tech-insights-backend/src/service/router.test.ts index 8a7b6bdbb3..0b7d3b7c45 100644 --- a/plugins/tech-insights-backend/src/service/router.test.ts +++ b/plugins/tech-insights-backend/src/service/router.test.ts @@ -19,7 +19,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import request from 'supertest'; import express from 'express'; -import { PersistenceContext } from './persistence/DatabaseManager'; +import { PersistenceContext } from './persistence/persistenceContext'; import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; import { DateTime } from 'luxon'; import { Knex } from 'knex'; diff --git a/plugins/tech-insights-backend/src/service/router.ts b/plugins/tech-insights-backend/src/service/router.ts index 752534928f..a8951318ea 100644 --- a/plugins/tech-insights-backend/src/service/router.ts +++ b/plugins/tech-insights-backend/src/service/router.ts @@ -25,7 +25,7 @@ import { import { CheckResult } from '@backstage/plugin-tech-insights-common'; import { Logger } from 'winston'; import { DateTime } from 'luxon'; -import { PersistenceContext } from './persistence/DatabaseManager'; +import { PersistenceContext } from './persistence/persistenceContext'; import { EntityRef, parseEntityName, diff --git a/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts b/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts index 7305f1778c..75e24b7c06 100644 --- a/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts +++ b/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts @@ -29,9 +29,9 @@ import { TechInsightCheck, } from '@backstage/plugin-tech-insights-node'; import { - DatabaseManager, + initializePersistenceContext, PersistenceContext, -} from './persistence/DatabaseManager'; +} from './persistence/persistenceContext'; import { CheckResult } from '@backstage/plugin-tech-insights-common'; /** @@ -105,7 +105,7 @@ export const buildTechInsightsContext = async < const factRetrieverRegistry = new FactRetrieverRegistry(factRetrievers); - const persistenceContext = await DatabaseManager.initializePersistenceContext( + const persistenceContext = await initializePersistenceContext( await database.getClient(), { logger }, ); diff --git a/plugins/tech-insights-node/api-report.md b/plugins/tech-insights-node/api-report.md new file mode 100644 index 0000000000..7a157309dd --- /dev/null +++ b/plugins/tech-insights-node/api-report.md @@ -0,0 +1,155 @@ +## API Report File for "@backstage/plugin-tech-insights-node" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { CheckResult } from '@backstage/plugin-tech-insights-common'; +import { Config } from '@backstage/config'; +import { DateTime } from 'luxon'; +import { Logger as Logger_2 } from 'winston'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; + +// @public +export type CheckValidationResponse = { + valid: boolean; + message?: string; + errors?: unknown[]; +}; + +// @public +export interface FactChecker< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + getChecks(): Promise; + runChecks(entity: string, checks?: string[]): Promise; + validate(check: CheckType): Promise; +} + +// @public +export interface FactCheckerFactory< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + // (undocumented) + construct( + repository: TechInsightsStore, + ): FactChecker; +} + +// @public +export interface FactRetriever { + // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag + // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" + // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag + // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" + entityFilter?: + | Record[] + | Record; + handler: (ctx: FactRetrieverContext) => Promise; + id: string; + schema: FactSchema; + version: string; +} + +// @public +export type FactRetrieverContext = { + config: Config; + discovery: PluginEndpointDiscovery; + logger: Logger_2; +}; + +// @public +export type FactRetrieverRegistration = { + factRetriever: FactRetriever; + cadence?: string; +}; + +// @public +export type FactSchema = { + [name: string]: { + type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; + description: string; + since?: string; + metadata?: Record; + }; +}; + +// Warning: (ae-missing-release-tag) "FactSchemaDefinition" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type FactSchemaDefinition = Omit; + +// @public +export type FlatTechInsightFact = TechInsightFact & { + id: string; +}; + +// @public +export interface TechInsightCheck { + description: string; + factIds: string[]; + failureMetadata?: Record; + id: string; + name: string; + successMetadata?: Record; + type: string; +} + +// @public +export interface TechInsightCheckRegistry { + // (undocumented) + get(checkId: string): Promise; + // (undocumented) + getAll(checks: string[]): Promise; + // (undocumented) + list(): Promise; + // (undocumented) + register(check: CheckType): Promise; +} + +// @public +export type TechInsightFact = { + entity: { + namespace: string; + kind: string; + name: string; + }; + facts: Record< + string, + | number + | string + | boolean + | DateTime + | number[] + | string[] + | boolean[] + | DateTime[] + >; + timestamp?: DateTime; +}; + +// @public +export interface TechInsightsStore { + getFactsBetweenTimestampsByIds( + ids: string[], + entity: string, + startDateTime: DateTime, + endDateTime: DateTime, + ): Promise<{ + [factRef: string]: FlatTechInsightFact[]; + }>; + // (undocumented) + getLatestFactsByIds( + ids: string[], + entity: string, + ): Promise<{ + [factRef: string]: FlatTechInsightFact; + }>; + getLatestSchemas(ids?: string[]): Promise; + insertFacts(id: string, facts: TechInsightFact[]): Promise; + insertFactSchema(schemaDefinition: FactSchemaDefinition): Promise; +} + +// (No @packageDocumentation comment for this package) +``` From 9c19d6ad0179834ebc3a9b34f1eac5f792dd36e5 Mon Sep 17 00:00:00 2001 From: Jussi Hallila Date: Fri, 29 Oct 2021 12:42:31 +0200 Subject: [PATCH 36/58] Tweak formatting of snippets on README.md files. Signed-off-by: Jussi Hallila --- .../README.md | 38 +++++------ plugins/tech-insights-backend/README.md | 68 ++++++++++++------- 2 files changed, 62 insertions(+), 44 deletions(-) diff --git a/plugins/tech-insights-backend-module-jsonfc/README.md b/plugins/tech-insights-backend-module-jsonfc/README.md index a19c3dd2ce..48c1e69919 100644 --- a/plugins/tech-insights-backend-module-jsonfc/README.md +++ b/plugins/tech-insights-backend-module-jsonfc/README.md @@ -17,32 +17,32 @@ yarn add @backstage/plugin-tech-insights-backend-module-jsonfc and modify the `techInsights.ts` file to contain a reference to the FactCheckers implementation. ```diff -+ import { JsonRulesEngineFactCheckerFactory } from '@backstage/plugin-tech-insights-backend-module-jsonfc'; ++import { JsonRulesEngineFactCheckerFactory } from '@backstage/plugin-tech-insights-backend-module-jsonfc'; -+ const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ -+ checks: [], -+ logger, -+ }), ++const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ ++ checks: [], ++ logger, ++}), - const builder = new DefaultTechInsightsBuilder({ - logger, - config, - database, - discovery, - factRetrievers: [myFactRetrieverRegistration], -+ factCheckerFactory: myFactCheckerFactory - }); + const builder = new DefaultTechInsightsBuilder({ + logger, + config, + database, + discovery, + factRetrievers: [myFactRetrieverRegistration], ++ factCheckerFactory: myFactCheckerFactory + }); ``` By default this implementation comes with an in-memory storage to store checks. You can inject an additional data store by adding an implementation of `TechInsightCheckRegistry` into the constructor options when creating a `JsonRulesEngineFactCheckerFactory`. That can be done as follows ```diff - const myTechInsightCheckRegistry: TechInsightCheckRegistry = // snip - const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ - checks: [], - logger, -+ checkRegistry: myTechInsightCheckRegistry - }), + const myTechInsightCheckRegistry: TechInsightCheckRegistry = // snip + const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ + checks: [], + logger, ++ checkRegistry: myTechInsightCheckRegistry + }), ``` diff --git a/plugins/tech-insights-backend/README.md b/plugins/tech-insights-backend/README.md index e7a78d34a2..e2cc1c0252 100644 --- a/plugins/tech-insights-backend/README.md +++ b/plugins/tech-insights-backend/README.md @@ -53,20 +53,20 @@ With the `techInsights.ts` router setup in place, add the router to `packages/backend/src/index.ts`: ```diff -+ import techInsights from './plugins/techInsights'; ++import techInsights from './plugins/techInsights'; - async function main() { - ... - const createEnv = makeCreateEnv(config); + async function main() { + ... + const createEnv = makeCreateEnv(config); - const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); -+ const techInsightsEnv = useHotMemoize(module, () => createEnv('tech_insights')); + const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); ++ const techInsightsEnv = useHotMemoize(module, () => createEnv('tech_insights')); - const apiRouter = Router(); -+ apiRouter.use('/tech-insights', await techInsights(techInsightsEnv)); - ... - apiRouter.use(notFoundHandler()); - } + const apiRouter = Router(); ++ apiRouter.use('/tech-insights', await techInsights(techInsightsEnv)); + ... + apiRouter.use(notFoundHandler()); + } ``` ### Adding fact retrievers @@ -104,14 +104,32 @@ const builder = new DefaultTechInsightsBuilder({ }); ``` +#### Running fact retrievers in a multi-instance installation + +Current logic on running scheduled fact retrievers is intended to be executed in a single instance. Running on multi-instane environment there might be some additional data accumulation when multiple fact retrievers would retrieve and persist their facts. To mitigate this it is recommended to mark a single instance to be a specific fact retriever instance. One way to do this is by using environment variables to indicate if the retrievers should be registered. This can be done for example like the code snippet below + +```diff +const builder = new DefaultTechInsightsBuilder({ + logger, + config, + database, + discovery, +- factRetrievers: [], ++ factRetrievers: process.env.MAIN_FACT_RETRIEVER_INSTANCE ? [myFactRetrieverRegistration] : [], +}); +``` + +Where the instance dedicated to handling retrieval of facts would have environment variable `MAIN_FACT_RETRIEVER_INSTANCE` set to true. + ### Creating Fact Retrievers -A Fact Retriever consist of four parts: +A Fact Retriever consist of four required and one optional parts: 1. `id` - unique identifier of a fact retriever 2. `version`: A semver string indicating the current version of the schema and the handler 3. `schema` - A versioned schema defining the shape of data a fact retriever returns 4. `handler` - An asynchronous function handling the logic of retrieving and returning facts for an entity +5. `entityFilter` - (Optional) EntityFilter object defining the entity kinds, types and/or names this fact retriever handles An example implementation of a FactRetriever could for example be as follows: @@ -178,21 +196,21 @@ yarn add @backstage/plugin-tech-insights-backend-module-jsonfc and modify the `techInsights.ts` file to contain a reference to the FactChecker implementation. ```diff -+ import { JsonRulesEngineFactCheckerFactory } from '@backstage/plugin-tech-insights-backend-module-jsonfc'; ++import { JsonRulesEngineFactCheckerFactory } from '@backstage/plugin-tech-insights-backend-module-jsonfc'; -+ const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ -+ checks: [], -+ logger, -+ }), ++const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ ++ checks: [], ++ logger, ++}), - const builder = new DefaultTechInsightsBuilder({ - logger, - config, - database, - discovery, - factRetrievers: [myFactRetrieverRegistration], -+ factCheckerFactory: myFactCheckerFactory - }); + const builder = new DefaultTechInsightsBuilder({ + logger, + config, + database, + discovery, + factRetrievers: [myFactRetrieverRegistration], ++ factCheckerFactory: myFactCheckerFactory + }); ``` To be able to run checks, you need to additionally add individual checks into your FactChecker implementation. For examples how to add these, you can check the documentation of the individual implementation of the FactChecker From b863feefbb48466b39cfecb023ef5112df60d753 Mon Sep 17 00:00:00 2001 From: Jussi Hallila Date: Fri, 29 Oct 2021 15:14:51 +0200 Subject: [PATCH 37/58] Address warnings within api-docs Move extra info on doc to be under remarks block Signed-off-by: Jussi Hallila --- .../src/service/fact/createFactRetriever.ts | 9 ++++++--- plugins/tech-insights-node/api-report.md | 8 +------- plugins/tech-insights-node/src/facts.ts | 10 ++++++++-- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts b/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts index b744d3b826..655736400a 100644 --- a/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts +++ b/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts @@ -23,6 +23,12 @@ import { * * A helper function to construct fact retriever registrations. * + * @param cadence - cron expression to indicate when the fact retriever should be triggered + * @param factRetriever - Implementation of fact retriever consisting of at least id, version, schema and handler + * + * + * @remarks + * * Cron expressions help: * ┌────────────── second (optional) # │ ┌──────────── minute @@ -34,9 +40,6 @@ import { # │ │ │ │ │ │ # * * * * * * * - * - * @param cadence - cron expression to indicate when the fact retriever should be triggered - * @param factRetriever - Implementation of fact retriever consisting of at least id, version, schema and handler */ export function createFactRetrieverRegistration( cadence: string, diff --git a/plugins/tech-insights-node/api-report.md b/plugins/tech-insights-node/api-report.md index 7a157309dd..caf4b4cf0e 100644 --- a/plugins/tech-insights-node/api-report.md +++ b/plugins/tech-insights-node/api-report.md @@ -39,10 +39,6 @@ export interface FactCheckerFactory< // @public export interface FactRetriever { - // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag - // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" - // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag - // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" entityFilter?: | Record[] | Record; @@ -75,9 +71,7 @@ export type FactSchema = { }; }; -// Warning: (ae-missing-release-tag) "FactSchemaDefinition" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export type FactSchemaDefinition = Omit; // @public diff --git a/plugins/tech-insights-node/src/facts.ts b/plugins/tech-insights-node/src/facts.ts index b912895ec6..a52a91f2f4 100644 --- a/plugins/tech-insights-node/src/facts.ts +++ b/plugins/tech-insights-node/src/facts.ts @@ -179,14 +179,20 @@ export interface FactRetriever { * If omitted, the retriever should apply to all entities. * * Should be defined for example: - * { field: 'kind', values: ['component'] } - * { field: 'metadata.name', values: ['component-1', 'component-2'] } + * \{ field: 'kind', values: \['component'\] \} + * \{ field: 'metadata.name', values: \['component-1', 'component-2'\] \} */ entityFilter?: | Record[] | Record; } +/** + * @public + * + * A flat serializable structure for Facts. + * Containing information about fact schema, version, id, and entity filters + */ export type FactSchemaDefinition = Omit; /** From 1bbaa719ff2d254571df03d870e1319029d46176 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 29 Oct 2021 16:10:13 +0200 Subject: [PATCH 38/58] user-settings: remove duplicate core-plugin-api dependency Signed-off-by: Patrik Oldsberg --- plugins/user-settings/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/user-settings/package.json b/plugins/user-settings/package.json index 9a3b65f7ae..9ec642892b 100644 --- a/plugins/user-settings/package.json +++ b/plugins/user-settings/package.json @@ -45,7 +45,6 @@ "devDependencies": { "@backstage/cli": "^0.8.1", "@backstage/core-app-api": "^0.1.19", - "@backstage/core-plugin-api": "^0.1.11", "@backstage/dev-utils": "^0.2.12", "@backstage/test-utils": "^0.1.20", "@testing-library/jest-dom": "^5.10.1", From dd355bca467bbcfa88fe7fec69a3f17abd70626f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 29 Oct 2021 16:14:25 +0200 Subject: [PATCH 39/58] cli: dynamically determine which packages are unsafe to repack Signed-off-by: Patrik Oldsberg --- .changeset/angry-countries-cheat.md | 5 +++++ packages/cli/src/lib/packager/index.ts | 10 ++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 .changeset/angry-countries-cheat.md diff --git a/.changeset/angry-countries-cheat.md b/.changeset/angry-countries-cheat.md new file mode 100644 index 0000000000..e97e0ae56c --- /dev/null +++ b/.changeset/angry-countries-cheat.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Switched to dynamically determining the packages that are unsafe to repack when executing the CLI within the Backstage main repo. diff --git a/packages/cli/src/lib/packager/index.ts b/packages/cli/src/lib/packager/index.ts index 6917795e83..dba42e6013 100644 --- a/packages/cli/src/lib/packager/index.ts +++ b/packages/cli/src/lib/packager/index.ts @@ -24,14 +24,16 @@ import { tmpdir } from 'os'; import tar, { CreateOptions } from 'tar'; import { paths } from '../paths'; import { run } from '../run'; -import { packageVersions } from '../version'; import { ParallelOption } from '../parallel'; +import { + dependencies as cliDependencies, + devDependencies as cliDevDependencies, +} from '../../../package.json'; // These packages aren't safe to pack in parallel since the CLI depends on them const UNSAFE_PACKAGES = [ - ...Object.keys(packageVersions), - '@backstage/cli-common', - '@backstage/config-loader', + ...Object.keys(cliDependencies), + ...Object.keys(cliDevDependencies), ]; type LernaPackage = { From f1903c71676b0c36ef448958d78720c18d2716b2 Mon Sep 17 00:00:00 2001 From: Samira Mokaram Date: Fri, 29 Oct 2021 16:47:02 +0200 Subject: [PATCH 40/58] add warning to dismissableBanner Signed-off-by: Samira Mokaram --- .../DismissableBanner/DismissableBanner.stories.tsx | 11 +++++++++++ .../DismissableBanner/DismissableBanner.tsx | 5 ++++- packages/theme/src/themes.ts | 1 + packages/theme/src/types.ts | 1 + 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/core-components/src/components/DismissableBanner/DismissableBanner.stories.tsx b/packages/core-components/src/components/DismissableBanner/DismissableBanner.stories.tsx index a64514be61..e0e9a89c9f 100644 --- a/packages/core-components/src/components/DismissableBanner/DismissableBanner.stories.tsx +++ b/packages/core-components/src/components/DismissableBanner/DismissableBanner.stories.tsx @@ -105,3 +105,14 @@ export const Fixed = () => ( ); +export const Warning = () => ( +
+ + + +
+); diff --git a/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx b/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx index 1213111c9d..40082925c6 100644 --- a/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx +++ b/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx @@ -79,12 +79,15 @@ const useStyles = makeStyles( error: { backgroundColor: theme.palette.banner.error, }, + warning: { + backgroundColor: theme.palette.banner.warning, + }, }), { name: 'BackstageDismissableBanner' }, ); type Props = { - variant: 'info' | 'error'; + variant: 'info' | 'error' | 'warning'; message: ReactNode; id: string; fixed?: boolean; diff --git a/packages/theme/src/themes.ts b/packages/theme/src/themes.ts index 3eb1bcc252..f35c45ef7d 100644 --- a/packages/theme/src/themes.ts +++ b/packages/theme/src/themes.ts @@ -55,6 +55,7 @@ export const lightTheme = createTheme({ error: '#E22134', text: '#FFFFFF', link: '#000000', + warning: '#FF9800', }, border: '#E6E6E6', textContrast: '#000000', diff --git a/packages/theme/src/types.ts b/packages/theme/src/types.ts index db3e89fa4b..40f45ee739 100644 --- a/packages/theme/src/types.ts +++ b/packages/theme/src/types.ts @@ -76,6 +76,7 @@ export type BackstagePaletteAdditions = { error: string; text: string; link: string; + warning: string; }; }; From c11a37710afb2d653d44a82ae9d6072923beabb7 Mon Sep 17 00:00:00 2001 From: Samira Mokaram Date: Fri, 29 Oct 2021 16:51:32 +0200 Subject: [PATCH 41/58] add changeset Signed-off-by: Samira Mokaram --- .changeset/dry-spies-cover.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/dry-spies-cover.md diff --git a/.changeset/dry-spies-cover.md b/.changeset/dry-spies-cover.md new file mode 100644 index 0000000000..3911fdde84 --- /dev/null +++ b/.changeset/dry-spies-cover.md @@ -0,0 +1,6 @@ +--- +'@backstage/core-components': patch +'@backstage/theme': patch +--- + +Will Add warning variant to Dismissable component. From 83cc72f6cbddce7a46b35353c9545081e2498d42 Mon Sep 17 00:00:00 2001 From: Samira Mokaram Date: Fri, 29 Oct 2021 17:02:48 +0200 Subject: [PATCH 42/58] fix Signed-off-by: Samira Mokaram --- .changeset/dry-spies-cover.md | 2 +- packages/theme/src/themes.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.changeset/dry-spies-cover.md b/.changeset/dry-spies-cover.md index 3911fdde84..91c6249d13 100644 --- a/.changeset/dry-spies-cover.md +++ b/.changeset/dry-spies-cover.md @@ -3,4 +3,4 @@ '@backstage/theme': patch --- -Will Add warning variant to Dismissable component. +Will Add warning variant to DismissableBanner component. diff --git a/packages/theme/src/themes.ts b/packages/theme/src/themes.ts index f35c45ef7d..7e99ce8e8a 100644 --- a/packages/theme/src/themes.ts +++ b/packages/theme/src/themes.ts @@ -130,6 +130,7 @@ export const darkTheme = createTheme({ error: '#E22134', text: '#FFFFFF', link: '#000000', + warning: '#FF9800', }, border: '#E6E6E6', textContrast: '#FFFFFF', From c493f261e2b48c3cbbacd4ecc1303859b919d4be Mon Sep 17 00:00:00 2001 From: Samira Mokaram Date: Fri, 29 Oct 2021 17:32:36 +0200 Subject: [PATCH 43/58] fix changeset Signed-off-by: Samira Mokaram --- .changeset/dry-spies-cover.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/dry-spies-cover.md b/.changeset/dry-spies-cover.md index 91c6249d13..7006dee5c4 100644 --- a/.changeset/dry-spies-cover.md +++ b/.changeset/dry-spies-cover.md @@ -3,4 +3,4 @@ '@backstage/theme': patch --- -Will Add warning variant to DismissableBanner component. +Will Add warning variant to `DismissableBanner` component. From a300f19a933b83a0a30e6b8279786eba94499719 Mon Sep 17 00:00:00 2001 From: Samira Mokaram Date: Fri, 29 Oct 2021 17:45:54 +0200 Subject: [PATCH 44/58] add api-report Signed-off-by: Samira Mokaram --- packages/theme/api-report.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/theme/api-report.md b/packages/theme/api-report.md index 72d7887214..30dcb7b040 100644 --- a/packages/theme/api-report.md +++ b/packages/theme/api-report.md @@ -64,6 +64,7 @@ export type BackstagePaletteAdditions = { error: string; text: string; link: string; + warning: string; }; }; From 15a205a804d1d4de26f1409d191192f3eb27f57f Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Oct 2021 16:18:15 +0200 Subject: [PATCH 45/58] chore: puppeteer! Signed-off-by: blam --- packages/e2e-test/package.json | 7 +++++-- packages/e2e-test/src/commands/run.ts | 14 ++++++++------ packages/e2e-test/src/lib/helpers.ts | 20 +++++++++++--------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/packages/e2e-test/package.json b/packages/e2e-test/package.json index e783fa1de6..06f13d7cbf 100644 --- a/packages/e2e-test/package.json +++ b/packages/e2e-test/package.json @@ -28,6 +28,7 @@ "@backstage/errors": "^0.1.2", "@types/fs-extra": "^9.0.1", "@types/node": "^14.14.32", + "@types/puppeteer": "^5.4.4", "chalk": "^4.0.0", "commander": "^6.1.0", "cross-fetch": "^3.0.6", @@ -35,12 +36,14 @@ "handlebars": "^4.7.3", "pgtools": "^0.3.0", "tree-kill": "^1.2.2", - "ts-node": "^10.0.0", - "zombie": "^6.1.4" + "ts-node": "^10.0.0" }, "nodemonConfig": { "watch": "./src", "exec": "bin/e2e-test", "ext": "ts" + }, + "dependencies": { + "puppeteer": "^10.4.0" } } diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts index b0db715f25..b91a65e72e 100644 --- a/packages/e2e-test/src/commands/run.ts +++ b/packages/e2e-test/src/commands/run.ts @@ -20,7 +20,8 @@ import fetch from 'cross-fetch'; import handlebars from 'handlebars'; import killTree from 'tree-kill'; import { resolve as resolvePath, join as joinPath } from 'path'; -import Browser from 'zombie'; +import puppeteer from 'puppeteer'; + import { spawnPiped, runPlain, @@ -350,17 +351,18 @@ async function testAppServe(pluginName: string, appDir: string) { GITHUB_TOKEN: 'abc', }, }); - Browser.localhost('localhost', 3000); + + const browser = await puppeteer.launch(); + const page = await browser.newPage(); + await page.goto('http://localhost:3000'); let successful = false; try { for (let attempts = 1; ; attempts++) { try { - const browser = new Browser(); - - await waitForPageWithText(browser, '/', 'My Company Catalog'); + await waitForPageWithText(page, '/', 'My Company Catalog'); await waitForPageWithText( - browser, + page, `/${pluginName}`, `Welcome to ${pluginName}!`, ); diff --git a/packages/e2e-test/src/lib/helpers.ts b/packages/e2e-test/src/lib/helpers.ts index 5193db08ec..ddb2714853 100644 --- a/packages/e2e-test/src/lib/helpers.ts +++ b/packages/e2e-test/src/lib/helpers.ts @@ -22,6 +22,7 @@ import { ChildProcess, } from 'child_process'; import { promisify } from 'util'; +import puppeteer from 'puppeteer'; const execFile = promisify(execFileCb); @@ -125,7 +126,7 @@ export async function waitForExit(child: ChildProcess) { } export async function waitForPageWithText( - browser: any, + page: puppeteer.Page, path: string, text: string, { intervalMs = 1000, maxFindAttempts = 50 } = {}, @@ -137,16 +138,17 @@ export async function waitForPageWithText( console.log(`Attempting to load page at ${path}, waiting ${waitTimeMs}`); await new Promise(resolve => setTimeout(resolve, waitTimeMs)); - await browser.visit(path); - - await new Promise(resolve => setTimeout(resolve, waitTimeMs)); + await page.goto(`http://localhost:3000${path}`); const escapedText = text.replace(/"|\\/g, '\\$&'); - browser.assert.evaluate( - `Array.from(document.querySelectorAll("*")).some(el => el.textContent === "${escapedText}")`, - true, - `expected to find text ${text}`, - ); + await expect(() => + page.evaluate(() => + Array.from(document.querySelectorAll('*')).some( + el => el.textContent === escapedText, + ), + ), + ).resolves.toBeTruthy(); + break; } catch (error) { assertError(error); From 1078543bc128b2f875e0b0a77bcf621e499ab095 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Oct 2021 16:20:39 +0200 Subject: [PATCH 46/58] chore: run the lock! Signed-off-by: blam --- yarn.lock | 320 +++++++++++++++++++----------------------------------- 1 file changed, 110 insertions(+), 210 deletions(-) diff --git a/yarn.lock b/yarn.lock index 64f9d5d995..3fec636e32 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7464,6 +7464,13 @@ resolved "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.4.tgz#fcf7205c25dff795ee79af1e30da2c9790808f11" integrity sha512-rZ5drC/jWjrArrS8BR6SIr4cWpW09RNTYt9AMZo3Jwwif+iacXAqgVjm0B0Bv/S1jhDXKHqRVNCbACkJ89RAnQ== +"@types/puppeteer@^5.4.4": + version "5.4.4" + resolved "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-5.4.4.tgz#e92abeccc4f46207c3e1b38934a1246be080ccd0" + integrity sha512-3Nau+qi69CN55VwZb0ATtdUAlYlqOOQ3OfQfq0Hqgc4JMFXiQT/XInlwQ9g6LbicDslE6loIFsXFklGh5XmI6Q== + dependencies: + "@types/node" "*" + "@types/q@^1.5.1": version "1.5.2" resolved "https://registry.npmjs.org/@types/q/-/q-1.5.2.tgz#690a1475b84f2a884fd07cd797c00f5f31356ea8" @@ -7973,6 +7980,13 @@ resolved "https://registry.npmjs.org/@types/yarnpkg__lockfile/-/yarnpkg__lockfile-1.1.4.tgz#445251eb00bd9c1e751f82c7c6bf4f714edfd464" integrity sha512-/emrKCfQMQmFCqRqqBJ0JueHBT06jBRM3e8OgnvDUcvuExONujIk2hFA5dNsN9Nt41ljGVDdChvCydATZ+KOZw== +"@types/yauzl@^2.9.1": + version "2.9.2" + resolved "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.2.tgz#c48e5d56aff1444409e39fa164b0b4d4552a7b7a" + integrity sha512-8uALY5LTvSuHgloDVUvWP3pIauILm+8/0pDMokuDYIoNsOkSwd5AiHBTSEJjKTDcZr5z8UpgOWZkxBF4iJftoA== + dependencies: + "@types/node" "*" + "@types/yup@^0.29.13": version "0.29.13" resolved "https://registry.npmjs.org/@types/yup/-/yup-0.29.13.tgz#21b137ba60841307a3c8a1050d3bf4e63ad561e9" @@ -8397,7 +8411,7 @@ a-sync-waterfall@^1.0.0: resolved "https://registry.npmjs.org/a-sync-waterfall/-/a-sync-waterfall-1.0.1.tgz#75b6b6aa72598b497a125e7a2770f14f4c8a1fa7" integrity sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA== -abab@^2.0.0, abab@^2.0.3: +abab@^2.0.3: version "2.0.3" resolved "https://registry.npmjs.org/abab/-/abab-2.0.3.tgz#623e2075e02eb2d3f2475e49f99c91846467907a" integrity sha512-tsFzPpcttalNjFBCFMqsKYQcWxxen1pgJR56by//QwvJc4/OUS3kPOOttx2tSIfjsylB0pYu7f5D3K1RCxUnUg== @@ -8427,14 +8441,6 @@ accepts@^1.3.5, accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.7: mime-types "~2.1.24" negotiator "0.6.2" -acorn-globals@^4.1.0: - version "4.3.4" - resolved "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.3.4.tgz#9fa1926addc11c97308c4e66d7add0d40c3272e7" - integrity sha512-clfQEh21R+D0leSbUdWf3OcfqyaCSAQ8Ryq00bofSekfr9W8u1jyYZo6ir0xu9Gtcf7BjcHJpnbZH7JOCpP60A== - dependencies: - acorn "^6.0.1" - acorn-walk "^6.0.1" - acorn-globals@^6.0.0: version "6.0.0" resolved "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz#46cdd39f0f8ff08a876619b55f5ac8a6dc770b45" @@ -8453,11 +8459,6 @@ acorn-jsx@^5.3.1: resolved "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.1.tgz#fc8661e11b7ac1539c47dbfea2e72b3af34d267b" integrity sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng== -acorn-walk@^6.0.1: - version "6.2.0" - resolved "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.2.0.tgz#123cb8f3b84c2171f1f7fb252615b1c78a6b1a8c" - integrity sha512-7evsyfH1cLOCdAzZAd43Cic04yKydNx0cF+7tiA19p1XnLLPU4dpCQOqpjqwokFe//vS0QqfqqjCS2JkiIs0cA== - acorn-walk@^7.1.1: version "7.1.1" resolved "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.1.1.tgz#345f0dffad5c735e7373d2fec9a1023e6a44b83e" @@ -8468,12 +8469,7 @@ acorn-walk@^8.1.1: resolved "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== -acorn@^5.5.3: - version "5.7.4" - resolved "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz#3e8d8a9947d0599a1796d10225d7432f4a4acf5e" - integrity sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg== - -acorn@^6.0.1, acorn@^6.4.1: +acorn@^6.4.1: version "6.4.1" resolved "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz#531e58ba3f51b9dacb9a6646ca4debf5b14ca474" integrity sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA== @@ -9044,11 +9040,6 @@ array-differ@^3.0.0: resolved "https://registry.npmjs.org/array-differ/-/array-differ-3.0.0.tgz#3cbb3d0f316810eafcc47624734237d6aee4ae6b" integrity sha512-THtfYS6KtME/yIAhKjZ2ul7XI96lQGHRputJQHO80LAWQnuGP4iCIN8vdMRboGbIEYBwU33q8Tch1os2+X0kMg== -array-equal@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz#8c2a5ef2472fd9ea742b04c77a75093ba2757c93" - integrity sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM= - array-filter@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/array-filter/-/array-filter-1.0.0.tgz#baf79e62e6ef4c2a4c0b831232daffec251f9d83" @@ -9211,11 +9202,6 @@ async-each@^1.0.1: resolved "https://registry.npmjs.org/async-each/-/async-each-1.0.3.tgz#b727dbf87d7651602f06f4d4ac387f47d91b0cbf" integrity sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ== -async-limiter@~1.0.0: - version "1.0.1" - resolved "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd" - integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ== - async-lock@^1.1.0: version "1.2.4" resolved "https://registry.npmjs.org/async-lock/-/async-lock-1.2.4.tgz#80d0d612383045dd0c30eb5aad08510c1397cb91" @@ -9646,7 +9632,7 @@ babel-preset-jest@^26.6.2: babel-plugin-jest-hoist "^26.6.2" babel-preset-current-node-syntax "^1.0.0" -babel-runtime@6.26.0, babel-runtime@^6.26.0: +babel-runtime@^6.26.0: version "6.26.0" resolved "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz#965c7058668e82b55d7bfe04ff2337bc8b5647fe" integrity sha1-llxwWGaOgrVde/4E/yM3vItWR/4= @@ -9863,7 +9849,7 @@ block-stream@*: dependencies: inherits "~2.0.0" -bluebird@3.7.2, bluebird@^3.3.5, bluebird@^3.5.1, bluebird@^3.5.5, bluebird@^3.7.2: +bluebird@3.7.2, bluebird@^3.3.5, bluebird@^3.5.5, bluebird@^3.7.2: version "3.7.2" resolved "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== @@ -10132,7 +10118,7 @@ buffer@4.9.2, buffer@^4.3.0: ieee754 "^1.1.4" isarray "^1.0.0" -buffer@^5.5.0, buffer@^5.7.0: +buffer@^5.2.1, buffer@^5.5.0, buffer@^5.7.0: version "5.7.1" resolved "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== @@ -11953,22 +11939,15 @@ csso@^4.0.2: dependencies: css-tree "1.0.0-alpha.37" -cssom@0.3.x, "cssom@>= 0.3.2 < 0.4.0", cssom@~0.3.6: - version "0.3.8" - resolved "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz#9f1276f5b2b463f2114d3f2c75250af8c1a36f4a" - integrity sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg== - cssom@^0.4.4: version "0.4.4" resolved "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz#5a66cf93d2d0b661d80bf6a44fb65f5c2e4e0a10" integrity sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw== -cssstyle@^1.0.0: - version "1.4.0" - resolved "https://registry.npmjs.org/cssstyle/-/cssstyle-1.4.0.tgz#9d31328229d3c565c61e586b02041a28fccdccf1" - integrity sha512-GBrLZYZ4X4x6/QEoBnIrqb8B/f5l4+8me2dkom/j1Gtbxy0kBv6OGzKuAsGM75bkGwGAFkt56Iwg28S3XTZgSA== - dependencies: - cssom "0.3.x" +cssom@~0.3.6: + version "0.3.8" + resolved "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz#9f1276f5b2b463f2114d3f2c75250af8c1a36f4a" + integrity sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg== cssstyle@^2.2.0: version "2.3.0" @@ -12332,15 +12311,6 @@ dashify@^2.0.0: resolved "https://registry.npmjs.org/dashify/-/dashify-2.0.0.tgz#fff270ca2868ca427fee571de35691d6e437a648" integrity sha512-hpA5C/YrPjucXypHPPc0oJ1l9Hf6wWbiOL7Ik42cxnsUOhWiCB/fylKbKqqJalW9FgkNQCw16YO8uW9Hs0Iy1A== -data-urls@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz#15ee0582baa5e22bb59c77140da8f9c76963bbfe" - integrity sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ== - dependencies: - abab "^2.0.0" - whatwg-mimetype "^2.2.0" - whatwg-url "^7.0.0" - data-urls@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz#156485a72963a970f5d5821aaf642bef2bf2db9b" @@ -12687,6 +12657,11 @@ detect-port@^1.3.0: address "^1.0.1" debug "^2.6.0" +devtools-protocol@0.0.901419: + version "0.0.901419" + resolved "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.901419.tgz#79b5459c48fe7e1c5563c02bd72f8fec3e0cebcd" + integrity sha512-4INMPwNm9XRpBukhNbF7OB6fNTTCaI8pzy/fXg0xQzAy5h3zL1P8xT3QazgKqBrb/hAYwIBizqDBZ7GtJE74QQ== + dezalgo@^1.0.0: version "1.0.3" resolved "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.3.tgz#7f742de066fc748bc8db820569dddce49bf0d456" @@ -12881,13 +12856,6 @@ domelementtype@^2.0.1, domelementtype@^2.1.0: resolved "https://registry.npmjs.org/domelementtype/-/domelementtype-2.1.0.tgz#a851c080a6d1c3d94344aed151d99f669edf585e" integrity sha512-LsTgx/L5VpD+Q8lmsXSHW2WpA+eBlZ9HPf3erD1IoPF00/3JKHZ3BknUVA2QGDNu69ZNmyFmCWBSO45XjYKC5w== -domexception@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz#937442644ca6a31261ef36e3ec677fe805582c90" - integrity sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug== - dependencies: - webidl-conversions "^4.0.2" - domexception@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz#fb44aefba793e1574b0af6aed2801d057529f304" @@ -13420,7 +13388,7 @@ escape-string-regexp@^5.0.0: resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz#4683126b500b61762f2dbebace1806e8be31b1c8" integrity sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw== -escodegen@^1.14.1, escodegen@^1.9.1: +escodegen@^1.14.1: version "1.14.3" resolved "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz#4e7b81fba61581dc97582ed78cab7f0e8d63f503" integrity sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw== @@ -13788,13 +13756,6 @@ events@^3.0.0, events@^3.2.0, events@^3.3.0: resolved "https://registry.npmjs.org/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== -eventsource@^1.0.5: - version "1.0.7" - resolved "https://registry.npmjs.org/eventsource/-/eventsource-1.0.7.tgz#8fbc72c93fcd34088090bc0a4e64f4b5cee6d8d0" - integrity sha512-4Ln17+vVT0k8aWq+t/bF5arcS3EpT9gYtW66EPacdj/mAFevznsnyoHLPy2BA8gbIQeIHoPsvwmfBftfcG//BQ== - dependencies: - original "^1.0.0" - evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz#7fcbdb198dc71959432efe13842684e0525acb02" @@ -14082,6 +14043,17 @@ extract-files@^8.0.0: resolved "https://registry.npmjs.org/extract-files/-/extract-files-8.1.0.tgz#46a0690d0fe77411a2e3804852adeaa65cd59288" integrity sha512-PTGtfthZK79WUMk+avLmwx3NGdU8+iVFXC2NMGxKsn0MnihOG2lvumj+AZo8CTwTrwjXDgZ5tztbRlEdRjBonQ== +extract-zip@2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" + integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== + dependencies: + debug "^4.1.1" + get-stream "^5.1.0" + yauzl "^2.10.0" + optionalDependencies: + "@types/yauzl" "^2.9.1" + extract-zip@^1.7.0: version "1.7.0" resolved "https://registry.npmjs.org/extract-zip/-/extract-zip-1.7.0.tgz#556cc3ae9df7f452c493a0cfb51cc30277940927" @@ -15874,13 +15846,6 @@ html-comment-regex@^1.1.0: resolved "https://registry.npmjs.org/html-comment-regex/-/html-comment-regex-1.1.2.tgz#97d4688aeb5c81886a364faa0cad1dda14d433a7" integrity sha512-P+M65QY2JQ5Y0G9KKdlDpo0zK+/OHptU5AaBwUfAIDJZk1MYf32Frm84EcOytfJE0t5JvkAnKlmjsXDnWzCJmQ== -html-encoding-sniffer@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz#e70d84b94da53aa375e11fe3a351be6642ca46f8" - integrity sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw== - dependencies: - whatwg-encoding "^1.0.1" - html-encoding-sniffer@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz#42a6dc4fd33f00281176e8b23759ca4e4fa185f3" @@ -16104,7 +16069,7 @@ https-browserify@^1.0.0: resolved "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73" integrity sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM= -https-proxy-agent@^5.0.0: +https-proxy-agent@5.0.0, https-proxy-agent@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== @@ -16149,7 +16114,7 @@ hyphenate-style-name@^1.0.2, hyphenate-style-name@^1.0.3: resolved "https://registry.npmjs.org/hyphenate-style-name/-/hyphenate-style-name-1.0.3.tgz#097bb7fa0b8f1a9cf0bd5c734cf95899981a9b48" integrity sha512-EcuixamT82oplpoJ2XU4pDtKGWQ7b00CD9f1ug9IaQ3p1bkHMiKCZ9ut9QDI6qsa6cpUuB+A/I+zLtdNK4n2DQ== -iconv-lite@0.4.24, iconv-lite@^0.4.21, iconv-lite@^0.4.24, iconv-lite@^0.4.4: +iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@^0.4.4: version "0.4.24" resolved "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== @@ -17939,38 +17904,6 @@ jscodeshift@^0.13.0: temp "^0.8.4" write-file-atomic "^2.3.0" -jsdom@11.12.0: - version "11.12.0" - resolved "https://registry.npmjs.org/jsdom/-/jsdom-11.12.0.tgz#1a80d40ddd378a1de59656e9e6dc5a3ba8657bc8" - integrity sha512-y8Px43oyiBM13Zc1z780FrfNLJCXTL40EWlty/LXUtcjykRBNgLlCjWXpfSPBl2iv+N7koQN+dvqszHZgT/Fjw== - dependencies: - abab "^2.0.0" - acorn "^5.5.3" - acorn-globals "^4.1.0" - array-equal "^1.0.0" - cssom ">= 0.3.2 < 0.4.0" - cssstyle "^1.0.0" - data-urls "^1.0.0" - domexception "^1.0.1" - escodegen "^1.9.1" - html-encoding-sniffer "^1.0.2" - left-pad "^1.3.0" - nwsapi "^2.0.7" - parse5 "4.0.0" - pn "^1.1.0" - request "^2.87.0" - request-promise-native "^1.0.5" - sax "^1.2.4" - symbol-tree "^3.2.2" - tough-cookie "^2.3.4" - w3c-hr-time "^1.0.1" - webidl-conversions "^4.0.2" - whatwg-encoding "^1.0.3" - whatwg-mimetype "^2.1.0" - whatwg-url "^6.4.1" - ws "^5.2.0" - xml-name-validator "^3.0.0" - jsdom@^16.4.0: version "16.4.0" resolved "https://registry.npmjs.org/jsdom/-/jsdom-16.4.0.tgz#36005bde2d136f73eee1a830c6d45e55408edddb" @@ -18581,11 +18514,6 @@ leasot@^12.0.0: strip-ansi "^6.0.0" text-table "^0.2.0" -left-pad@^1.3.0: - version "1.3.0" - resolved "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz#5b8a3a7765dfe001261dde915589e782f8c94d1e" - integrity sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA== - lerna@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/lerna/-/lerna-4.0.0.tgz#b139d685d50ea0ca1be87713a7c2f44a5b678e9e" @@ -20198,7 +20126,7 @@ mime@1.6.0, mime@^1.4.1: resolved "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== -mime@^2.2.0, mime@^2.3.1, mime@^2.4.4, mime@^2.4.6: +mime@^2.2.0, mime@^2.4.4, mime@^2.4.6: version "2.5.2" resolved "https://registry.npmjs.org/mime/-/mime-2.5.2.tgz#6e3dc6cc2b9510643830e5f19d5cb753da5eeabe" integrity sha512-tqkh47FzKeCPD2PUiPB6pkbMzsCasjxAfC62/Wap5qrUWcb+sFasXUC5I3gYM5iBM8v/Qpn4UK0x+j0iHyFPDg== @@ -21170,7 +21098,7 @@ nunjucks@^3.2.3: asap "^2.0.3" commander "^5.1.0" -nwsapi@^2.0.7, nwsapi@^2.2.0: +nwsapi@^2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz#204879a9e3d068ff2a55139c2c772780681a38b7" integrity sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ== @@ -21433,13 +21361,6 @@ ora@^5.3.0, ora@^5.4.1: strip-ansi "^6.0.0" wcwidth "^1.0.1" -original@^1.0.0: - version "1.0.2" - resolved "https://registry.npmjs.org/original/-/original-1.0.2.tgz#e442a61cffe1c5fd20a65f3261c26663b303f25f" - integrity sha512-hyBVl6iqqUOJ8FqRe+l/gS8H+kKYjrEndd5Pm1MfBtsEKA038HkkdbAl/72EAXGyonD/PFsvmVG+EvcIpliMBg== - dependencies: - url-parse "^1.4.3" - os-browserify@^0.3.0: version "0.3.0" resolved "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz#854373c7f5c2315914fc9bfc6bd8238fdda1ec27" @@ -21878,11 +21799,6 @@ parse-url@^5.0.0: parse-path "^4.0.0" protocols "^1.4.0" -parse5@4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz#6d78656e3da8d78b4ec0b906f7c08ef1dfe3f608" - integrity sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA== - parse5@5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178" @@ -22364,6 +22280,13 @@ pirates@^4.0.0, pirates@^4.0.1: dependencies: node-modules-regexp "^1.0.0" +pkg-dir@4.2.0, pkg-dir@^4.1.0, pkg-dir@^4.2.0: + version "4.2.0" + resolved "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" + integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== + dependencies: + find-up "^4.0.0" + pkg-dir@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz#f6d5d1109e19d63edf428e0bd57e12777615334b" @@ -22378,13 +22301,6 @@ pkg-dir@^3.0.0: dependencies: find-up "^3.0.0" -pkg-dir@^4.1.0, pkg-dir@^4.2.0: - version "4.2.0" - resolved "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" - integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== - dependencies: - find-up "^4.0.0" - pkg-dir@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/pkg-dir/-/pkg-dir-5.0.0.tgz#a02d6aebe6ba133a928f74aec20bafdfe6b8e760" @@ -22416,11 +22332,6 @@ pluralize@^8.0.0: resolved "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1" integrity sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA== -pn@^1.1.0: - version "1.1.0" - resolved "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz#e2f4cef0e219f463c179ab37463e4e1ecdccbafb" - integrity sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA== - pnp-webpack-plugin@1.6.4: version "1.6.4" resolved "https://registry.npmjs.org/pnp-webpack-plugin/-/pnp-webpack-plugin-1.6.4.tgz#c9711ac4dc48a685dabafc86f8b6dd9f8df84149" @@ -23017,6 +22928,11 @@ process@^0.11.10: resolved "https://registry.npmjs.org/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" integrity sha1-czIwDoQBYb2j5podHZGn1LwW8YI= +progress@2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/progress/-/progress-2.0.1.tgz#c9242169342b1c29d275889c95734621b1952e31" + integrity sha512-OE+a6vzqazc+K6LxJrX5UPyKFvGnL5CYmq2jFGNIBWHpc4QyE49/YOumcrpQFJpfejmvRtbJzgO1zPmMCqlbBg== + progress@^2.0.0: version "2.0.3" resolved "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" @@ -23182,6 +23098,11 @@ proxy-addr@~2.0.5: forwarded "~0.1.2" ipaddr.js "1.9.1" +proxy-from-env@1.1.0: + version "1.1.0" + resolved "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + prr@~1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" @@ -23282,6 +23203,24 @@ pupa@^2.0.1: dependencies: escape-goat "^2.0.0" +puppeteer@^10.4.0: + version "10.4.0" + resolved "https://registry.npmjs.org/puppeteer/-/puppeteer-10.4.0.tgz#a6465ff97fda0576c4ac29601406f67e6fea3dc7" + integrity sha512-2cP8mBoqnu5gzAVpbZ0fRaobBWZM8GEUF4I1F6WbgHrKV/rz7SX8PG2wMymZgD0wo0UBlg2FBPNxlF/xlqW6+w== + dependencies: + debug "4.3.1" + devtools-protocol "0.0.901419" + extract-zip "2.0.1" + https-proxy-agent "5.0.0" + node-fetch "2.6.1" + pkg-dir "4.2.0" + progress "2.0.1" + proxy-from-env "1.1.0" + rimraf "3.0.2" + tar-fs "2.0.0" + unbzip2-stream "1.3.3" + ws "7.4.6" + q@^1.1.2, q@^1.5.1: version "1.5.1" resolved "https://registry.npmjs.org/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" @@ -24587,7 +24526,7 @@ request-promise-core@1.1.3: dependencies: lodash "^4.17.15" -request-promise-native@^1.0.5, request-promise-native@^1.0.8: +request-promise-native@^1.0.8: version "1.0.8" resolved "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.8.tgz#a455b960b826e44e2bf8999af64dff2bfe58cb36" integrity sha512-dapwLGqkHtwL5AEbfenuzjTYg35Jd6KPytsC2/TLkVMz8rm+tNt72MGUWT1RP/aYawMpN6HqbNGBQaRcBtjQMQ== @@ -24596,7 +24535,7 @@ request-promise-native@^1.0.5, request-promise-native@^1.0.8: stealthy-require "^1.1.1" tough-cookie "^2.3.3" -request@^2.85.0, request@^2.87.0, request@^2.88.0, request@^2.88.2: +request@^2.87.0, request@^2.88.0, request@^2.88.2: version "2.88.2" resolved "https://registry.npmjs.org/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== @@ -24814,7 +24753,7 @@ rimraf@2, rimraf@^2.2.8, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.3: dependencies: glob "^7.1.3" -rimraf@^3.0.0, rimraf@^3.0.2: +rimraf@3.0.2, rimraf@^3.0.0, rimraf@^3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== @@ -26585,7 +26524,7 @@ symbol-observable@1.2.0, symbol-observable@^1.0.4, symbol-observable@^1.1.0, sym resolved "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz#c22688aed4eab3cdc2dfeacbb561660560a00804" integrity sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ== -symbol-tree@^3.2.2, symbol-tree@^3.2.4: +symbol-tree@^3.2.4: version "3.2.4" resolved "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== @@ -26628,6 +26567,16 @@ tapable@^2.0.0, tapable@^2.1.1, tapable@^2.2.0: resolved "https://registry.npmjs.org/tapable/-/tapable-2.2.0.tgz#5c373d281d9c672848213d0e037d1c4165ab426b" integrity sha512-FBk4IesMV1rBxX2tfiK8RAmogtWn53puLOQlvO8XuwlgxcYbP4mVPS9Ph4aeamSyyVjOl24aYWAuc8U5kCVwMw== +tar-fs@2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.0.tgz#677700fc0c8b337a78bee3623fdc235f21d7afad" + integrity sha512-vaY0obB6Om/fso8a8vakQBzwholQ7v5+uy+tF3Ozvxv1KNezmVQAiWtcNmMHFSFPqL3dJA8ha6gdtFbfX9mcxA== + dependencies: + chownr "^1.1.1" + mkdirp "^0.5.1" + pump "^3.0.0" + tar-stream "^2.0.0" + tar-fs@^2.0.0, tar-fs@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" @@ -27109,7 +27058,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -tough-cookie@^2.3.3, tough-cookie@^2.3.4, tough-cookie@~2.5.0: +tough-cookie@^2.3.3, tough-cookie@~2.5.0: version "2.5.0" resolved "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== @@ -27135,13 +27084,6 @@ tough-cookie@^4.0.0: punycode "^2.1.1" universalify "^0.1.2" -tr46@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz#a8b13fd6bfd2489519674ccde55ba3693b706d09" - integrity sha1-qLE/1r/SSJUZZ0zN5VujaTtwbQk= - dependencies: - punycode "^2.1.0" - tr46@^2.0.2: version "2.0.2" resolved "https://registry.npmjs.org/tr46/-/tr46-2.0.2.tgz#03273586def1595ae08fedb38d7733cee91d2479" @@ -27565,6 +27507,14 @@ unbox-primitive@^1.0.0: has-symbols "^1.0.0" which-boxed-primitive "^1.0.1" +unbzip2-stream@1.3.3: + version "1.3.3" + resolved "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.3.3.tgz#d156d205e670d8d8c393e1c02ebd506422873f6a" + integrity sha512-fUlAF7U9Ah1Q6EieQ4x4zLNejrRvDWUYmxXUpN3uziFYCHapjWFaCAnreY9bGgxzaMCFAPPpYNng57CypwJVhg== + dependencies: + buffer "^5.2.1" + through "^2.3.8" + unc-path-regex@^0.1.2: version "0.1.2" resolved "https://registry.npmjs.org/unc-path-regex/-/unc-path-regex-0.1.2.tgz#e73dd3d7b0d7c5ed86fbac6b0ae7d8c6a69d50fa" @@ -27962,7 +27912,7 @@ url-parse-lax@^3.0.0: dependencies: prepend-http "^2.0.0" -url-parse@^1.4.3, url-parse@^1.5.3: +url-parse@^1.5.3: version "1.5.3" resolved "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz#71c1303d38fb6639ade183c2992c8cc0686df862" integrity sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ== @@ -28291,7 +28241,7 @@ vscode-languageserver-types@^3.15.1: resolved "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.15.1.tgz#17be71d78d2f6236d414f0001ce1ef4d23e6b6de" integrity sha512-+a9MPUQrNGRrGU630OGbYVQ+11iOIovjCkqxajPa9w57Sd5ruK8WQNsslzpa0x/QJqC8kRc2DUxWjIFwoNm4ZQ== -w3c-hr-time@^1.0.1, w3c-hr-time@^1.0.2: +w3c-hr-time@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd" integrity sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ== @@ -28390,11 +28340,6 @@ web-streams-polyfill@4.0.0-beta.1: resolved "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.1.tgz#3b19b9817374b7cee06d374ba7eeb3aeb80e8c95" integrity sha512-3ux37gEX670UUphBF9AMCq8XM6iQ8Ac6A+DSRRjDoRBm1ufCkaCDdNVbaqq60PsEkdNlLKrGtv/YBP4EJXqNtQ== -webidl-conversions@^4.0.2: - version "4.0.2" - resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad" - integrity sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg== - webidl-conversions@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff" @@ -28588,7 +28533,7 @@ websocket-extensions@>=0.1.1: resolved "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz#7f8473bc839dfd87608adb95d7eb075211578a42" integrity sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg== -whatwg-encoding@^1.0.1, whatwg-encoding@^1.0.3, whatwg-encoding@^1.0.5: +whatwg-encoding@^1.0.5: version "1.0.5" resolved "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz#5abacf777c32166a51d085d6b4f3e7d27113ddb0" integrity sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw== @@ -28605,29 +28550,11 @@ whatwg-fetch@^3.4.1: resolved "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.4.1.tgz#e5f871572d6879663fa5674c8f833f15a8425ab3" integrity sha512-sofZVzE1wKwO+EYPbWfiwzaKovWiZXf4coEzjGP9b2GBVgQRLQUZ2QcuPpQExGDAW5GItpEm6Tl4OU5mywnAoQ== -whatwg-mimetype@^2.1.0, whatwg-mimetype@^2.2.0, whatwg-mimetype@^2.3.0: +whatwg-mimetype@^2.3.0: version "2.3.0" resolved "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== -whatwg-url@^6.4.1: - version "6.5.0" - resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz#f2df02bff176fd65070df74ad5ccbb5a199965a8" - integrity sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ== - dependencies: - lodash.sortby "^4.7.0" - tr46 "^1.0.1" - webidl-conversions "^4.0.2" - -whatwg-url@^7.0.0: - version "7.1.0" - resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz#c2c492f1eca612988efd3d2266be1b9fc6170d06" - integrity sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg== - dependencies: - lodash.sortby "^4.7.0" - tr46 "^1.0.1" - webidl-conversions "^4.0.2" - whatwg-url@^8.0.0, whatwg-url@^8.4.0: version "8.4.0" resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.4.0.tgz#50fb9615b05469591d2b2bd6dfaed2942ed72837" @@ -28869,25 +28796,16 @@ ws@7.4.5, ws@^7.4.6: resolved "https://registry.npmjs.org/ws/-/ws-7.5.5.tgz#8b4bc4af518cfabd0473ae4f99144287b33eb881" integrity sha512-BAkMFcAzl8as1G/hArkxOxq3G7pjUqQ3gzYbLL0/5zNkph70e+lCoxBGnm6AW1+/aiNeV4fnKqZ8m4GZewmH2w== -ws@^5.2.0: - version "5.2.3" - resolved "https://registry.npmjs.org/ws/-/ws-5.2.3.tgz#05541053414921bc29c63bee14b8b0dd50b07b3d" - integrity sha512-jZArVERrMsKUatIdnLzqvcfydI85dvd/Fp1u/VOpfdDWQ4c9qWXe+VIeAbQ5FrDwciAkr+lzofXLz3Kuf26AOA== - dependencies: - async-limiter "~1.0.0" +ws@7.4.6: + version "7.4.6" + resolved "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c" + integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A== "ws@^5.2.0 || ^6.0.0 || ^7.0.0", ws@^7.2.3, ws@^7.3.1: version "7.5.0" resolved "https://registry.npmjs.org/ws/-/ws-7.5.0.tgz#0033bafea031fb9df041b2026fc72a571ca44691" integrity sha512-6ezXvzOZupqKj4jUqbQ9tXuJNo+BR2gU8fFRk3XCP3e0G6WT414u5ELe6Y0vtp7kmSJ3F7YWObSNr1ESsgi4vw== -ws@^6.1.2: - version "6.2.1" - resolved "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz#442fdf0a47ed64f59b6a5d8ff130f4748ed524fb" - integrity sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA== - dependencies: - async-limiter "~1.0.0" - ws@^8.1.0: version "8.2.3" resolved "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz#63a56456db1b04367d0b721a0b80cae6d8becbba" @@ -29260,24 +29178,6 @@ zip-stream@^4.1.0: compress-commons "^4.1.0" readable-stream "^3.6.0" -zombie@^6.1.4: - version "6.1.4" - resolved "https://registry.npmjs.org/zombie/-/zombie-6.1.4.tgz#9f0f53f3d9a032beb7f3fe5b382146a3475a4d47" - integrity sha512-yxNvKtyz3PP8lkr31AYh7vdbBD4is9hYXiOQKPp+k/7GiDiFQXX1Ex+peCl4ttodu/bHZcIluJ8lxMla5XefBQ== - dependencies: - babel-runtime "6.26.0" - bluebird "^3.5.1" - debug "^4.1.0" - eventsource "^1.0.5" - iconv-lite "^0.4.21" - jsdom "11.12.0" - lodash "^4.17.10" - mime "^2.3.1" - ms "^2.1.1" - request "^2.85.0" - tough-cookie "^2.3.4" - ws "^6.1.2" - zwitch@^1.0.0: version "1.0.5" resolved "https://registry.npmjs.org/zwitch/-/zwitch-1.0.5.tgz#d11d7381ffed16b742f6af7b3f223d5cd9fe9920" From e3ef16366edf6f1a9cb47f6472d84b38d4799103 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Oct 2021 16:23:43 +0200 Subject: [PATCH 47/58] chore: setup chrome in the e2e containers Signed-off-by: blam --- .github/workflows/e2e-win.yml | 2 ++ .github/workflows/e2e.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/e2e-win.yml b/.github/workflows/e2e-win.yml index acaf4e189e..36b3f75578 100644 --- a/.github/workflows/e2e-win.yml +++ b/.github/workflows/e2e-win.yml @@ -40,6 +40,8 @@ jobs: node-version: ${{ matrix.node-version }} - name: Add msbuild to PATH uses: microsoft/setup-msbuild@v1.0.2 + - name: setup chrome + uses: browser-actions/setup-chrome@latest - name: yarn install run: yarn install --frozen-lockfile diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d7754a0046..0967eeeacc 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -53,6 +53,8 @@ jobs: id: yarn-cache if: steps.cache-modules.outputs.cache-hit != 'true' run: echo "::set-output name=dir::$(yarn cache dir)" + - name: setup chrome + uses: browser-actions/setup-chrome@latest - name: cache global yarn cache uses: actions/cache@v2 if: steps.cache-modules.outputs.cache-hit != 'true' From 4b2740332c127ba245df6341ea4e68b9f1f5c6dc Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Oct 2021 18:06:47 +0200 Subject: [PATCH 48/58] chore: now we have puppeteer! Signed-off-by: blam --- packages/e2e-test/package.json | 4 +--- packages/e2e-test/src/commands/run.ts | 11 +++++++---- packages/e2e-test/src/lib/helpers.ts | 14 ++++++++------ 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/e2e-test/package.json b/packages/e2e-test/package.json index 06f13d7cbf..1b8e321edb 100644 --- a/packages/e2e-test/package.json +++ b/packages/e2e-test/package.json @@ -35,6 +35,7 @@ "fs-extra": "9.1.0", "handlebars": "^4.7.3", "pgtools": "^0.3.0", + "puppeteer": "^10.4.0", "tree-kill": "^1.2.2", "ts-node": "^10.0.0" }, @@ -42,8 +43,5 @@ "watch": "./src", "exec": "bin/e2e-test", "ext": "ts" - }, - "dependencies": { - "puppeteer": "^10.4.0" } } diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts index b91a65e72e..9eb32e378c 100644 --- a/packages/e2e-test/src/commands/run.ts +++ b/packages/e2e-test/src/commands/run.ts @@ -352,14 +352,16 @@ async function testAppServe(pluginName: string, appDir: string) { }, }); - const browser = await puppeteer.launch(); - const page = await browser.newPage(); - await page.goto('http://localhost:3000'); - let successful = false; + + let browser; try { for (let attempts = 1; ; attempts++) { try { + browser = await puppeteer.launch(); + const page = await browser.newPage(); + await page.goto('http://localhost:3000'); + await waitForPageWithText(page, '/', 'My Company Catalog'); await waitForPageWithText( page, @@ -380,6 +382,7 @@ async function testAppServe(pluginName: string, appDir: string) { } } finally { // Kill entire process group, otherwise we'll end up with hanging serve processes + if (browser) await browser.close(); killTree(startApp.pid); } diff --git a/packages/e2e-test/src/lib/helpers.ts b/packages/e2e-test/src/lib/helpers.ts index ddb2714853..b3212f3698 100644 --- a/packages/e2e-test/src/lib/helpers.ts +++ b/packages/e2e-test/src/lib/helpers.ts @@ -141,13 +141,15 @@ export async function waitForPageWithText( await page.goto(`http://localhost:3000${path}`); const escapedText = text.replace(/"|\\/g, '\\$&'); - await expect(() => - page.evaluate(() => - Array.from(document.querySelectorAll('*')).some( - el => el.textContent === escapedText, - ), + const match = await page.evaluate(() => + Array.from(document.querySelectorAll('*')).some( + el => el.textContent === escapedText, ), - ).resolves.toBeTruthy(); + ); + + if (!match) { + throw new Error(`Expected to find text ${escapedText}`); + } break; } catch (error) { From 03f368f740a3a5e1c25c721f43d5ba799c67318b Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Oct 2021 18:30:33 +0200 Subject: [PATCH 49/58] chore: finally made it work Signed-off-by: blam --- packages/e2e-test/src/commands/run.ts | 3 ++- packages/e2e-test/src/lib/helpers.ts | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts index 9eb32e378c..eedaca9a8a 100644 --- a/packages/e2e-test/src/commands/run.ts +++ b/packages/e2e-test/src/commands/run.ts @@ -360,7 +360,8 @@ async function testAppServe(pluginName: string, appDir: string) { try { browser = await puppeteer.launch(); const page = await browser.newPage(); - await page.goto('http://localhost:3000'); + + await page.goto('http://localhost:3000', { waitUntil: 'networkidle0' }); await waitForPageWithText(page, '/', 'My Company Catalog'); await waitForPageWithText( diff --git a/packages/e2e-test/src/lib/helpers.ts b/packages/e2e-test/src/lib/helpers.ts index b3212f3698..db8861e111 100644 --- a/packages/e2e-test/src/lib/helpers.ts +++ b/packages/e2e-test/src/lib/helpers.ts @@ -138,7 +138,9 @@ export async function waitForPageWithText( console.log(`Attempting to load page at ${path}, waiting ${waitTimeMs}`); await new Promise(resolve => setTimeout(resolve, waitTimeMs)); - await page.goto(`http://localhost:3000${path}`); + await page.goto(`http://localhost:3000${path}`, { + waitUntil: 'networkidle0', + }); const escapedText = text.replace(/"|\\/g, '\\$&'); const match = await page.evaluate(() => From bb5fe02de3d3b08b6eda6ee403248358fe3ead66 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Oct 2021 18:57:42 +0200 Subject: [PATCH 50/58] chore: don't need escaped text now as it's actually a function call like all good javascripts Signed-off-by: blam --- packages/e2e-test/src/lib/helpers.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/e2e-test/src/lib/helpers.ts b/packages/e2e-test/src/lib/helpers.ts index db8861e111..f51cf5b798 100644 --- a/packages/e2e-test/src/lib/helpers.ts +++ b/packages/e2e-test/src/lib/helpers.ts @@ -142,15 +142,14 @@ export async function waitForPageWithText( waitUntil: 'networkidle0', }); - const escapedText = text.replace(/"|\\/g, '\\$&'); const match = await page.evaluate(() => Array.from(document.querySelectorAll('*')).some( - el => el.textContent === escapedText, + el => el.textContent === text, ), ); if (!match) { - throw new Error(`Expected to find text ${escapedText}`); + throw new Error(`Expected to find text ${text}`); } break; From fc6c990cc9c9ccb34c2438ce33243243d5966c06 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Oct 2021 19:31:53 +0200 Subject: [PATCH 51/58] chore: this time, I promise. I fixed it. Sorted out all the scoping stuff for the hacky stuff that evaluate does with .toString Signed-off-by: blam --- packages/e2e-test/src/lib/helpers.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/e2e-test/src/lib/helpers.ts b/packages/e2e-test/src/lib/helpers.ts index f51cf5b798..350898234d 100644 --- a/packages/e2e-test/src/lib/helpers.ts +++ b/packages/e2e-test/src/lib/helpers.ts @@ -142,10 +142,12 @@ export async function waitForPageWithText( waitUntil: 'networkidle0', }); - const match = await page.evaluate(() => - Array.from(document.querySelectorAll('*')).some( - el => el.textContent === text, - ), + const match = await page.evaluate( + textContent => + Array.from(document.querySelectorAll('*')).some( + el => el.textContent === textContent, + ), + text, ); if (!match) { @@ -154,6 +156,7 @@ export async function waitForPageWithText( break; } catch (error) { + console.log(error); assertError(error); findAttempts++; From 614fd12ad0eeb20b130433e726e8e2264f903602 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Oct 2021 20:49:47 +0200 Subject: [PATCH 52/58] chore: wrap up the killTree? Signed-off-by: blam --- packages/core-components/package.json | 4 +- packages/core-components/package.json-prepack | 92 +++++++++++++++++++ packages/e2e-test/src/commands/run.ts | 8 +- 3 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 packages/core-components/package.json-prepack diff --git a/packages/core-components/package.json b/packages/core-components/package.json index 7f50af7488..5534d27c2a 100644 --- a/packages/core-components/package.json +++ b/packages/core-components/package.json @@ -18,8 +18,8 @@ "backstage" ], "license": "Apache-2.0", - "main": "src/index.ts", - "types": "src/index.ts", + "main": "dist/index.esm.js", + "types": "dist/index.d.ts", "scripts": { "build": "backstage-cli build --outputs types,esm", "lint": "backstage-cli lint", diff --git a/packages/core-components/package.json-prepack b/packages/core-components/package.json-prepack new file mode 100644 index 0000000000..038b99ca90 --- /dev/null +++ b/packages/core-components/package.json-prepack @@ -0,0 +1,92 @@ +{ + "name": "@backstage/core-components", + "description": "Core components used by Backstage plugins and apps", + "version": "0.7.1", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.esm.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "packages/core-components" + }, + "keywords": [ + "backstage" + ], + "license": "Apache-2.0", + "main": "src/index.ts", + "types": "src/index.ts", + "scripts": { + "build": "backstage-cli build --outputs types,esm", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/config": "^0.1.10", + "@backstage/core-plugin-api": "^0.1.11", + "@backstage/errors": "^0.1.3", + "@backstage/theme": "^0.2.11", + "@material-table/core": "^3.1.0", + "@material-ui/core": "^4.12.2", + "@material-ui/icons": "^4.9.1", + "@material-ui/lab": "4.0.0-alpha.57", + "@types/react": "*", + "@types/react-sparklines": "^1.7.0", + "@types/react-text-truncate": "^0.14.0", + "classnames": "^2.2.6", + "clsx": "^1.1.0", + "d3-selection": "^3.0.0", + "d3-shape": "^3.0.0", + "d3-zoom": "^3.0.0", + "dagre": "^0.8.5", + "immer": "^9.0.1", + "lodash": "^4.17.21", + "pluralize": "^8.0.0", + "prop-types": "^15.7.2", + "qs": "^6.9.4", + "rc-progress": "^3.0.0", + "react": "^16.12.0", + "react-dom": "^16.12.0", + "react-helmet": "6.1.0", + "react-hook-form": "^7.12.2", + "react-markdown": "^7.0.1", + "react-router": "6.0.0-beta.0", + "react-router-dom": "6.0.0-beta.0", + "react-sparklines": "^1.7.0", + "react-syntax-highlighter": "^15.4.3", + "react-text-truncate": "^0.16.0", + "react-use": "^17.2.4", + "remark-gfm": "^2.0.0", + "zen-observable": "^0.8.15" + }, + "devDependencies": { + "@backstage/core-app-api": "^0.1.18", + "@backstage/cli": "^0.8.0", + "@backstage/test-utils": "^0.1.19", + "@testing-library/jest-dom": "^5.10.1", + "@testing-library/react": "^11.2.5", + "@testing-library/react-hooks": "^7.0.2", + "@testing-library/user-event": "^13.1.8", + "@types/classnames": "^2.2.9", + "@types/d3-selection": "^3.0.1", + "@types/d3-shape": "^3.0.1", + "@types/d3-zoom": "^3.0.1", + "@types/dagre": "^0.7.44", + "@types/google-protobuf": "^3.7.2", + "@types/jest": "^26.0.7", + "@types/node": "^14.14.32", + "@types/react-helmet": "^6.1.0", + "@types/react-syntax-highlighter": "^13.5.2", + "@types/zen-observable": "^0.8.0" + }, + "files": [ + "dist" + ] +} diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts index eedaca9a8a..e5399c13f9 100644 --- a/packages/e2e-test/src/commands/run.ts +++ b/packages/e2e-test/src/commands/run.ts @@ -384,7 +384,9 @@ async function testAppServe(pluginName: string, appDir: string) { } finally { // Kill entire process group, otherwise we'll end up with hanging serve processes if (browser) await browser.close(); - killTree(startApp.pid); + await new Promise((res, rej) => + killTree(startApp.pid, err => (err ? rej(err) : res())), + ); } try { @@ -463,7 +465,9 @@ async function testBackendStart(appDir: string, isPostgres: boolean) { } finally { print('Stopping the child process'); // Kill entire process group, otherwise we'll end up with hanging serve processes - killTree(child.pid); + await new Promise((res, rej) => + killTree(child.pid, err => (err ? rej(err) : res())), + ); } try { From 9e3b76f86a1208837a9363015256447c396beb6c Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Oct 2021 21:01:08 +0200 Subject: [PATCH 53/58] chore: revert some accidental additions Signed-off-by: blam i --- packages/core-components/package.json | 4 +- packages/core-components/package.json-prepack | 92 ------------------- 2 files changed, 2 insertions(+), 94 deletions(-) delete mode 100644 packages/core-components/package.json-prepack diff --git a/packages/core-components/package.json b/packages/core-components/package.json index 5534d27c2a..7f50af7488 100644 --- a/packages/core-components/package.json +++ b/packages/core-components/package.json @@ -18,8 +18,8 @@ "backstage" ], "license": "Apache-2.0", - "main": "dist/index.esm.js", - "types": "dist/index.d.ts", + "main": "src/index.ts", + "types": "src/index.ts", "scripts": { "build": "backstage-cli build --outputs types,esm", "lint": "backstage-cli lint", diff --git a/packages/core-components/package.json-prepack b/packages/core-components/package.json-prepack deleted file mode 100644 index 038b99ca90..0000000000 --- a/packages/core-components/package.json-prepack +++ /dev/null @@ -1,92 +0,0 @@ -{ - "name": "@backstage/core-components", - "description": "Core components used by Backstage plugins and apps", - "version": "0.7.1", - "private": false, - "publishConfig": { - "access": "public", - "main": "dist/index.esm.js", - "types": "dist/index.d.ts" - }, - "homepage": "https://backstage.io", - "repository": { - "type": "git", - "url": "https://github.com/backstage/backstage", - "directory": "packages/core-components" - }, - "keywords": [ - "backstage" - ], - "license": "Apache-2.0", - "main": "src/index.ts", - "types": "src/index.ts", - "scripts": { - "build": "backstage-cli build --outputs types,esm", - "lint": "backstage-cli lint", - "test": "backstage-cli test", - "prepack": "backstage-cli prepack", - "postpack": "backstage-cli postpack", - "clean": "backstage-cli clean" - }, - "dependencies": { - "@backstage/config": "^0.1.10", - "@backstage/core-plugin-api": "^0.1.11", - "@backstage/errors": "^0.1.3", - "@backstage/theme": "^0.2.11", - "@material-table/core": "^3.1.0", - "@material-ui/core": "^4.12.2", - "@material-ui/icons": "^4.9.1", - "@material-ui/lab": "4.0.0-alpha.57", - "@types/react": "*", - "@types/react-sparklines": "^1.7.0", - "@types/react-text-truncate": "^0.14.0", - "classnames": "^2.2.6", - "clsx": "^1.1.0", - "d3-selection": "^3.0.0", - "d3-shape": "^3.0.0", - "d3-zoom": "^3.0.0", - "dagre": "^0.8.5", - "immer": "^9.0.1", - "lodash": "^4.17.21", - "pluralize": "^8.0.0", - "prop-types": "^15.7.2", - "qs": "^6.9.4", - "rc-progress": "^3.0.0", - "react": "^16.12.0", - "react-dom": "^16.12.0", - "react-helmet": "6.1.0", - "react-hook-form": "^7.12.2", - "react-markdown": "^7.0.1", - "react-router": "6.0.0-beta.0", - "react-router-dom": "6.0.0-beta.0", - "react-sparklines": "^1.7.0", - "react-syntax-highlighter": "^15.4.3", - "react-text-truncate": "^0.16.0", - "react-use": "^17.2.4", - "remark-gfm": "^2.0.0", - "zen-observable": "^0.8.15" - }, - "devDependencies": { - "@backstage/core-app-api": "^0.1.18", - "@backstage/cli": "^0.8.0", - "@backstage/test-utils": "^0.1.19", - "@testing-library/jest-dom": "^5.10.1", - "@testing-library/react": "^11.2.5", - "@testing-library/react-hooks": "^7.0.2", - "@testing-library/user-event": "^13.1.8", - "@types/classnames": "^2.2.9", - "@types/d3-selection": "^3.0.1", - "@types/d3-shape": "^3.0.1", - "@types/d3-zoom": "^3.0.1", - "@types/dagre": "^0.7.44", - "@types/google-protobuf": "^3.7.2", - "@types/jest": "^26.0.7", - "@types/node": "^14.14.32", - "@types/react-helmet": "^6.1.0", - "@types/react-syntax-highlighter": "^13.5.2", - "@types/zen-observable": "^0.8.0" - }, - "files": [ - "dist" - ] -} From 8535bbd53c1061adf145b562b08e7bbd7408dc55 Mon Sep 17 00:00:00 2001 From: Ben Lambert Date: Thu, 28 Oct 2021 21:30:52 +0200 Subject: [PATCH 54/58] Update run.ts Signed-off-by: Ben Lambert --- packages/e2e-test/src/commands/run.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts index e5399c13f9..1ea51a9265 100644 --- a/packages/e2e-test/src/commands/run.ts +++ b/packages/e2e-test/src/commands/run.ts @@ -383,7 +383,6 @@ async function testAppServe(pluginName: string, appDir: string) { } } finally { // Kill entire process group, otherwise we'll end up with hanging serve processes - if (browser) await browser.close(); await new Promise((res, rej) => killTree(startApp.pid, err => (err ? rej(err) : res())), ); From 0f254f45a5489cef9fc38e0584e6ce26dd83db3c Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 29 Oct 2021 11:04:42 +0200 Subject: [PATCH 55/58] chore: removing postgres env for now Signed-off-by: blam --- .github/workflows/e2e.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0967eeeacc..7ddd8c7ca4 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -74,8 +74,4 @@ jobs: run: | sudo sysctl fs.inotify.max_user_watches=524288 yarn e2e-test run - env: - POSTGRES_HOST: localhost - POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres + From fbbc73ac7dc385f5c7b2dfecaa48c93ab490fd41 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 29 Oct 2021 11:42:22 +0200 Subject: [PATCH 56/58] chore: fixing prettier Signed-off-by: blam --- .github/workflows/e2e.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 7ddd8c7ca4..287ef91f16 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -74,4 +74,3 @@ jobs: run: | sudo sysctl fs.inotify.max_user_watches=524288 yarn e2e-test run - From 08724f0bc710b1639d91f164e56724f8af393347 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 29 Oct 2021 18:06:14 +0200 Subject: [PATCH 57/58] chore: actually check the text in stderr to make sure that it's something that we don't expect Signed-off-by: blam --- packages/e2e-test/src/commands/run.ts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts index 1ea51a9265..8f4b267b94 100644 --- a/packages/e2e-test/src/commands/run.ts +++ b/packages/e2e-test/src/commands/run.ts @@ -445,9 +445,26 @@ async function testBackendStart(appDir: string, isPostgres: boolean) { }); let successful = false; + const stdErrorHasErrors = (input: string) => { + const lines = input.split('\n').filter(Boolean); + return ( + lines.filter( + l => + !l.includes('Use of deprecated folder mapping') && + !l.includes('Update this package.json to use a subpath') && + !l.includes( + '(Use `node --trace-deprecation ...` to show where the warning was created)', + ), + ).length !== 0 + ); + }; + try { - await waitFor(() => stdout.includes('Listening on ') || stderr !== ''); + await waitFor( + () => stdout.includes('Listening on ') || stdErrorHasErrors(stderr), + ); if (stderr !== '') { + print(`Expected stderr to be clean, got ${stderr}`); // Skipping the whole block throw new Error(stderr); } @@ -460,6 +477,7 @@ async function testBackendStart(appDir: string, isPostgres: boolean) { print('Entities fetched successfully'); successful = true; } catch (error) { + print(''); throw new Error(`Backend failed to startup: ${error}`); } finally { print('Stopping the child process'); From 5b43ae3b5e8557f4cd6cf5612789b440df51662d Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 29 Oct 2021 18:23:41 +0200 Subject: [PATCH 58/58] chore: finally got it working now right? Signed-off-by: blam --- packages/e2e-test/src/commands/run.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts index 8f4b267b94..e1d8473121 100644 --- a/packages/e2e-test/src/commands/run.ts +++ b/packages/e2e-test/src/commands/run.ts @@ -377,7 +377,7 @@ async function testAppServe(pluginName: string, appDir: string) { if (attempts >= 20) { throw new Error(`App serve test failed, ${error}`); } - console.log(`App serve failed, trying again, ${error}`); + print(`App serve failed, trying again, ${error}`); await new Promise(resolve => setTimeout(resolve, 1000)); } } @@ -463,7 +463,7 @@ async function testBackendStart(appDir: string, isPostgres: boolean) { await waitFor( () => stdout.includes('Listening on ') || stdErrorHasErrors(stderr), ); - if (stderr !== '') { + if (stdErrorHasErrors(stderr)) { print(`Expected stderr to be clean, got ${stderr}`); // Skipping the whole block throw new Error(stderr);