diff --git a/.changeset/angry-countries-cheat.md b/.changeset/angry-countries-cheat.md new file mode 100644 index 0000000000..e97e0ae56c --- /dev/null +++ b/.changeset/angry-countries-cheat.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Switched to dynamically determining the packages that are unsafe to repack when executing the CLI within the Backstage main repo. diff --git a/.changeset/dry-spies-cover.md b/.changeset/dry-spies-cover.md new file mode 100644 index 0000000000..7006dee5c4 --- /dev/null +++ b/.changeset/dry-spies-cover.md @@ -0,0 +1,6 @@ +--- +'@backstage/core-components': patch +'@backstage/theme': patch +--- + +Will Add warning variant to `DismissableBanner` component. diff --git a/.changeset/long-bugs-kiss.md b/.changeset/long-bugs-kiss.md new file mode 100644 index 0000000000..49e0edae72 --- /dev/null +++ b/.changeset/long-bugs-kiss.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-kafka': patch +--- + +Use IdentityApi to provide Auth Token for KafkaBackendClient Api calls diff --git a/.changeset/loud-bats-flow.md b/.changeset/loud-bats-flow.md new file mode 100644 index 0000000000..af4837dbbe --- /dev/null +++ b/.changeset/loud-bats-flow.md @@ -0,0 +1,5 @@ +--- +'@backstage/test-utils': patch +--- + +Update Keyboard deprecation with a link to the recommended successor diff --git a/.changeset/lovely-cars-sneeze.md b/.changeset/lovely-cars-sneeze.md new file mode 100644 index 0000000000..8e6bfd691c --- /dev/null +++ b/.changeset/lovely-cars-sneeze.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Expose some classes and interfaces public so TaskWorkers can run externally from the scaffolder API. diff --git a/.changeset/techdocs-buh-bump-ts.md b/.changeset/techdocs-buh-bump-ts.md new file mode 100644 index 0000000000..438843e04b --- /dev/null +++ b/.changeset/techdocs-buh-bump-ts.md @@ -0,0 +1,5 @@ +--- +'@backstage/techdocs-common': patch +--- + +Default TechDocs container used at docs generation-time is now [v0.3.5](https://github.com/backstage/techdocs-container/releases/tag/v0.3.5). diff --git a/.github/workflows/e2e-win.yml b/.github/workflows/e2e-win.yml index acaf4e189e..36b3f75578 100644 --- a/.github/workflows/e2e-win.yml +++ b/.github/workflows/e2e-win.yml @@ -40,6 +40,8 @@ jobs: node-version: ${{ matrix.node-version }} - name: Add msbuild to PATH uses: microsoft/setup-msbuild@v1.0.2 + - name: setup chrome + uses: browser-actions/setup-chrome@latest - name: yarn install run: yarn install --frozen-lockfile diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d7754a0046..287ef91f16 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -53,6 +53,8 @@ jobs: id: yarn-cache if: steps.cache-modules.outputs.cache-hit != 'true' run: echo "::set-output name=dir::$(yarn cache dir)" + - name: setup chrome + uses: browser-actions/setup-chrome@latest - name: cache global yarn cache uses: actions/cache@v2 if: steps.cache-modules.outputs.cache-hit != 'true' @@ -72,8 +74,3 @@ jobs: run: | sudo sysctl fs.inotify.max_user_watches=524288 yarn e2e-test run - env: - POSTGRES_HOST: localhost - POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index c174b94557..7100e86ed0 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -502,6 +502,8 @@ spec: lifecycle: production owner: artist-relations-team system: artist-engagement-portal + dependsOn: + - resource:default/artists-db providesApis: - artist-api ``` diff --git a/packages/backend/package.json b/packages/backend/package.json index b9a28c2853..f1d58252a4 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -48,6 +48,9 @@ "@backstage/plugin-search-backend-module-elasticsearch": "^0.0.4", "@backstage/plugin-search-backend-module-pg": "^0.2.1", "@backstage/plugin-techdocs-backend": "^0.10.5", + "@backstage/plugin-tech-insights-backend": "^0.1.0", + "@backstage/plugin-tech-insights-node": "^0.1.0", + "@backstage/plugin-tech-insights-backend-module-jsonfc": "^0.1.0", "@backstage/plugin-todo-backend": "^0.1.13", "@gitbeaker/node": "^30.2.0", "@octokit/rest": "^18.5.3", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index b6c148dfba..ffdce949b7 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -53,6 +53,7 @@ import graphql from './plugins/graphql'; import app from './plugins/app'; import badges from './plugins/badges'; import jenkins from './plugins/jenkins'; +import techInsights from './plugins/techInsights'; import { PluginEnvironment } from './types'; function makeCreateEnv(config: Config) { @@ -107,12 +108,16 @@ async function main() { const appEnv = useHotMemoize(module, () => createEnv('app')); const badgesEnv = useHotMemoize(module, () => createEnv('badges')); const jenkinsEnv = useHotMemoize(module, () => createEnv('jenkins')); + const techInsightsEnv = useHotMemoize(module, () => + createEnv('tech-insights'), + ); const apiRouter = Router(); apiRouter.use('/catalog', await catalog(catalogEnv)); apiRouter.use('/code-coverage', await codeCoverage(codeCoverageEnv)); apiRouter.use('/rollbar', await rollbar(rollbarEnv)); apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)); + apiRouter.use('/tech-insights', await techInsights(techInsightsEnv)); apiRouter.use('/auth', await auth(authEnv)); apiRouter.use('/azure-devops', await azureDevOps(azureDevOpsEnv)); apiRouter.use('/search', await search(searchEnv)); diff --git a/packages/backend/src/plugins/techInsights.ts b/packages/backend/src/plugins/techInsights.ts new file mode 100644 index 0000000000..abab1d133d --- /dev/null +++ b/packages/backend/src/plugins/techInsights.ts @@ -0,0 +1,107 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + createRouter, + buildTechInsightsContext, + createFactRetrieverRegistration, +} from '@backstage/plugin-tech-insights-backend'; +import { Router } from 'express'; +import { PluginEnvironment } from '../types'; +import { CatalogClient } from '@backstage/catalog-client'; +import { + JsonRulesEngineFactCheckerFactory, + JSON_RULE_ENGINE_CHECK_TYPE, +} from '@backstage/plugin-tech-insights-backend-module-jsonfc'; + +export default async function createPlugin({ + logger, + config, + discovery, + database, +}: PluginEnvironment): Promise { + const techInsightsContext = await buildTechInsightsContext({ + logger, + config, + database, + discovery, + factRetrievers: [ + createFactRetrieverRegistration('5 4 * * 6', { + // Example cron, At 04:05 on Saturday. + id: 'testRetriever', + version: '1.1.2', + entityFilter: [{ kind: 'component' }], // EntityFilter to be used in the future (creating checks, graphs etc.) to figure out which entities this fact retrieves data for. + schema: { + examplenumberfact: { + type: 'integer', + description: 'Example fact returning a number', + }, + }, + handler: async _ctx => { + const catalogClient = new CatalogClient({ + discoveryApi: discovery, + }); + const entities = await catalogClient.getEntities({ + filter: [{ kind: 'component' }], + }); + + return Promise.resolve( + entities.items.map(it => { + return { + entity: { + namespace: it.metadata.namespace!!, + kind: it.kind, + name: it.metadata.name, + }, + facts: { + examplenumberfact: 2, + }, + }; + }), + ); + }, + }), + ], + factCheckerFactory: new JsonRulesEngineFactCheckerFactory({ + checks: [ + { + id: 'simpleTestCheck', + type: JSON_RULE_ENGINE_CHECK_TYPE, + name: 'simpleTestCheck', + description: 'Simple Check For Testing', + factIds: ['testRetriever'], + rule: { + conditions: { + all: [ + { + fact: 'examplenumberfact', + operator: 'lessThan', + value: 5, + }, + ], + }, + }, + }, + ], + logger, + }), + }); + + return await createRouter({ + ...techInsightsContext, + logger, + config, + }); +} diff --git a/packages/cli/src/lib/packager/index.ts b/packages/cli/src/lib/packager/index.ts index 6917795e83..dba42e6013 100644 --- a/packages/cli/src/lib/packager/index.ts +++ b/packages/cli/src/lib/packager/index.ts @@ -24,14 +24,16 @@ import { tmpdir } from 'os'; import tar, { CreateOptions } from 'tar'; import { paths } from '../paths'; import { run } from '../run'; -import { packageVersions } from '../version'; import { ParallelOption } from '../parallel'; +import { + dependencies as cliDependencies, + devDependencies as cliDevDependencies, +} from '../../../package.json'; // These packages aren't safe to pack in parallel since the CLI depends on them const UNSAFE_PACKAGES = [ - ...Object.keys(packageVersions), - '@backstage/cli-common', - '@backstage/config-loader', + ...Object.keys(cliDependencies), + ...Object.keys(cliDevDependencies), ]; type LernaPackage = { diff --git a/packages/core-components/src/components/DismissableBanner/DismissableBanner.stories.tsx b/packages/core-components/src/components/DismissableBanner/DismissableBanner.stories.tsx index a64514be61..e0e9a89c9f 100644 --- a/packages/core-components/src/components/DismissableBanner/DismissableBanner.stories.tsx +++ b/packages/core-components/src/components/DismissableBanner/DismissableBanner.stories.tsx @@ -105,3 +105,14 @@ export const Fixed = () => ( ); +export const Warning = () => ( +
+ + + +
+); diff --git a/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx b/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx index 1213111c9d..40082925c6 100644 --- a/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx +++ b/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx @@ -79,12 +79,15 @@ const useStyles = makeStyles( error: { backgroundColor: theme.palette.banner.error, }, + warning: { + backgroundColor: theme.palette.banner.warning, + }, }), { name: 'BackstageDismissableBanner' }, ); type Props = { - variant: 'info' | 'error'; + variant: 'info' | 'error' | 'warning'; message: ReactNode; id: string; fixed?: boolean; diff --git a/packages/core-components/src/layout/HeaderActionMenu/HeaderActionMenu.test.tsx b/packages/core-components/src/layout/HeaderActionMenu/HeaderActionMenu.test.tsx index b1bd398d15..58785c5c90 100644 --- a/packages/core-components/src/layout/HeaderActionMenu/HeaderActionMenu.test.tsx +++ b/packages/core-components/src/layout/HeaderActionMenu/HeaderActionMenu.test.tsx @@ -15,13 +15,10 @@ */ import React from 'react'; -import { render, fireEvent } from '@testing-library/react'; -import { - wrapInTestApp, - Keyboard, - renderInTestApp, -} from '@backstage/test-utils'; +import { fireEvent } from '@testing-library/react'; +import { renderInTestApp } from '@backstage/test-utils'; import { HeaderActionMenu } from './HeaderActionMenu'; +import userEvent from '@testing-library/user-event'; describe('', () => { it('renders without any items and without exploding', async () => { @@ -89,16 +86,13 @@ describe('', () => { }); it('should close when hitting escape', async () => { - const rendered = render( - wrapInTestApp( - , - ), + const rendered = await renderInTestApp( + , ); - expect(rendered.container.getAttribute('aria-hidden')).toBeNull(); fireEvent.click(rendered.getByTestId('header-action-menu')); expect(rendered.container.getAttribute('aria-hidden')).toBe('true'); - await Keyboard.type(rendered, ''); + userEvent.type(rendered.getByTestId('header-action-menu'), '{esc}'); expect(rendered.container.getAttribute('aria-hidden')).toBeNull(); }); }); diff --git a/packages/create-app/CHANGELOG.md b/packages/create-app/CHANGELOG.md index 05e067ce2c..95d99d0532 100644 --- a/packages/create-app/CHANGELOG.md +++ b/packages/create-app/CHANGELOG.md @@ -1,5 +1,7 @@ # @backstage/create-app +## 0.4.2 + ## 0.4.1 ### Patch Changes diff --git a/packages/create-app/package.json b/packages/create-app/package.json index 725f865435..ad4ab8dabd 100644 --- a/packages/create-app/package.json +++ b/packages/create-app/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/create-app", "description": "A CLI that helps you create your own Backstage app", - "version": "0.4.1", + "version": "0.4.2", "private": false, "publishConfig": { "access": "public" diff --git a/packages/e2e-test/package.json b/packages/e2e-test/package.json index e783fa1de6..1b8e321edb 100644 --- a/packages/e2e-test/package.json +++ b/packages/e2e-test/package.json @@ -28,15 +28,16 @@ "@backstage/errors": "^0.1.2", "@types/fs-extra": "^9.0.1", "@types/node": "^14.14.32", + "@types/puppeteer": "^5.4.4", "chalk": "^4.0.0", "commander": "^6.1.0", "cross-fetch": "^3.0.6", "fs-extra": "9.1.0", "handlebars": "^4.7.3", "pgtools": "^0.3.0", + "puppeteer": "^10.4.0", "tree-kill": "^1.2.2", - "ts-node": "^10.0.0", - "zombie": "^6.1.4" + "ts-node": "^10.0.0" }, "nodemonConfig": { "watch": "./src", diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts index b0db715f25..e1d8473121 100644 --- a/packages/e2e-test/src/commands/run.ts +++ b/packages/e2e-test/src/commands/run.ts @@ -20,7 +20,8 @@ import fetch from 'cross-fetch'; import handlebars from 'handlebars'; import killTree from 'tree-kill'; import { resolve as resolvePath, join as joinPath } from 'path'; -import Browser from 'zombie'; +import puppeteer from 'puppeteer'; + import { spawnPiped, runPlain, @@ -350,17 +351,21 @@ async function testAppServe(pluginName: string, appDir: string) { GITHUB_TOKEN: 'abc', }, }); - Browser.localhost('localhost', 3000); let successful = false; + + let browser; try { for (let attempts = 1; ; attempts++) { try { - const browser = new Browser(); + browser = await puppeteer.launch(); + const page = await browser.newPage(); - await waitForPageWithText(browser, '/', 'My Company Catalog'); + await page.goto('http://localhost:3000', { waitUntil: 'networkidle0' }); + + await waitForPageWithText(page, '/', 'My Company Catalog'); await waitForPageWithText( - browser, + page, `/${pluginName}`, `Welcome to ${pluginName}!`, ); @@ -372,13 +377,15 @@ async function testAppServe(pluginName: string, appDir: string) { if (attempts >= 20) { throw new Error(`App serve test failed, ${error}`); } - console.log(`App serve failed, trying again, ${error}`); + print(`App serve failed, trying again, ${error}`); await new Promise(resolve => setTimeout(resolve, 1000)); } } } finally { // Kill entire process group, otherwise we'll end up with hanging serve processes - killTree(startApp.pid); + await new Promise((res, rej) => + killTree(startApp.pid, err => (err ? rej(err) : res())), + ); } try { @@ -438,9 +445,26 @@ async function testBackendStart(appDir: string, isPostgres: boolean) { }); let successful = false; + const stdErrorHasErrors = (input: string) => { + const lines = input.split('\n').filter(Boolean); + return ( + lines.filter( + l => + !l.includes('Use of deprecated folder mapping') && + !l.includes('Update this package.json to use a subpath') && + !l.includes( + '(Use `node --trace-deprecation ...` to show where the warning was created)', + ), + ).length !== 0 + ); + }; + try { - await waitFor(() => stdout.includes('Listening on ') || stderr !== ''); - if (stderr !== '') { + await waitFor( + () => stdout.includes('Listening on ') || stdErrorHasErrors(stderr), + ); + if (stdErrorHasErrors(stderr)) { + print(`Expected stderr to be clean, got ${stderr}`); // Skipping the whole block throw new Error(stderr); } @@ -453,11 +477,14 @@ async function testBackendStart(appDir: string, isPostgres: boolean) { print('Entities fetched successfully'); successful = true; } catch (error) { + print(''); throw new Error(`Backend failed to startup: ${error}`); } finally { print('Stopping the child process'); // Kill entire process group, otherwise we'll end up with hanging serve processes - killTree(child.pid); + await new Promise((res, rej) => + killTree(child.pid, err => (err ? rej(err) : res())), + ); } try { diff --git a/packages/e2e-test/src/lib/helpers.ts b/packages/e2e-test/src/lib/helpers.ts index 5193db08ec..350898234d 100644 --- a/packages/e2e-test/src/lib/helpers.ts +++ b/packages/e2e-test/src/lib/helpers.ts @@ -22,6 +22,7 @@ import { ChildProcess, } from 'child_process'; import { promisify } from 'util'; +import puppeteer from 'puppeteer'; const execFile = promisify(execFileCb); @@ -125,7 +126,7 @@ export async function waitForExit(child: ChildProcess) { } export async function waitForPageWithText( - browser: any, + page: puppeteer.Page, path: string, text: string, { intervalMs = 1000, maxFindAttempts = 50 } = {}, @@ -137,18 +138,25 @@ export async function waitForPageWithText( console.log(`Attempting to load page at ${path}, waiting ${waitTimeMs}`); await new Promise(resolve => setTimeout(resolve, waitTimeMs)); - await browser.visit(path); + await page.goto(`http://localhost:3000${path}`, { + waitUntil: 'networkidle0', + }); - await new Promise(resolve => setTimeout(resolve, waitTimeMs)); - - const escapedText = text.replace(/"|\\/g, '\\$&'); - browser.assert.evaluate( - `Array.from(document.querySelectorAll("*")).some(el => el.textContent === "${escapedText}")`, - true, - `expected to find text ${text}`, + const match = await page.evaluate( + textContent => + Array.from(document.querySelectorAll('*')).some( + el => el.textContent === textContent, + ), + text, ); + + if (!match) { + throw new Error(`Expected to find text ${text}`); + } + break; } catch (error) { + console.log(error); assertError(error); findAttempts++; diff --git a/packages/techdocs-common/api-report.md b/packages/techdocs-common/api-report.md index f5a00c25ea..61371525e5 100644 --- a/packages/techdocs-common/api-report.md +++ b/packages/techdocs-common/api-report.md @@ -256,7 +256,7 @@ export class TechdocsGenerator implements GeneratorBase { config: Config; scmIntegrations: ScmIntegrationRegistry; }); - static readonly defaultDockerImage = 'spotify/techdocs:v0.3.4'; + static readonly defaultDockerImage = 'spotify/techdocs:v0.3.5'; // (undocumented) static fromConfig( config: Config, diff --git a/packages/techdocs-common/src/stages/generate/techdocs.ts b/packages/techdocs-common/src/stages/generate/techdocs.ts index e87278c2d4..728cab5a81 100644 --- a/packages/techdocs-common/src/stages/generate/techdocs.ts +++ b/packages/techdocs-common/src/stages/generate/techdocs.ts @@ -44,7 +44,7 @@ export class TechdocsGenerator implements GeneratorBase { * The default docker image (and version) used to generate content. Public * and static so that techdocs-common consumers can use the same version. */ - public static readonly defaultDockerImage = 'spotify/techdocs:v0.3.4'; + public static readonly defaultDockerImage = 'spotify/techdocs:v0.3.5'; private readonly logger: Logger; private readonly containerRunner: ContainerRunner; private readonly options: GeneratorConfig; diff --git a/packages/test-utils/src/testUtils/Keyboard.js b/packages/test-utils/src/testUtils/Keyboard.js index 3f4724d562..2a7928f8cd 100644 --- a/packages/test-utils/src/testUtils/Keyboard.js +++ b/packages/test-utils/src/testUtils/Keyboard.js @@ -25,7 +25,7 @@ const codes = { /** * @public - * @deprecated because it has no usages. Perhaps resurfaced in the future when need be. + * @deprecated superseded by {@link @testing-library/user-event#userEvent} */ export class Keyboard { static async type(target, input) { diff --git a/packages/theme/api-report.md b/packages/theme/api-report.md index 72d7887214..30dcb7b040 100644 --- a/packages/theme/api-report.md +++ b/packages/theme/api-report.md @@ -64,6 +64,7 @@ export type BackstagePaletteAdditions = { error: string; text: string; link: string; + warning: string; }; }; diff --git a/packages/theme/src/themes.ts b/packages/theme/src/themes.ts index 3eb1bcc252..7e99ce8e8a 100644 --- a/packages/theme/src/themes.ts +++ b/packages/theme/src/themes.ts @@ -55,6 +55,7 @@ export const lightTheme = createTheme({ error: '#E22134', text: '#FFFFFF', link: '#000000', + warning: '#FF9800', }, border: '#E6E6E6', textContrast: '#000000', @@ -129,6 +130,7 @@ export const darkTheme = createTheme({ error: '#E22134', text: '#FFFFFF', link: '#000000', + warning: '#FF9800', }, border: '#E6E6E6', textContrast: '#FFFFFF', diff --git a/packages/theme/src/types.ts b/packages/theme/src/types.ts index db3e89fa4b..40f45ee739 100644 --- a/packages/theme/src/types.ts +++ b/packages/theme/src/types.ts @@ -76,6 +76,7 @@ export type BackstagePaletteAdditions = { error: string; text: string; link: string; + warning: string; }; }; diff --git a/plugins/kafka/src/api/KafkaBackendClient.ts b/plugins/kafka/src/api/KafkaBackendClient.ts index 988939ff1a..aea5736aad 100644 --- a/plugins/kafka/src/api/KafkaBackendClient.ts +++ b/plugins/kafka/src/api/KafkaBackendClient.ts @@ -15,21 +15,28 @@ */ import { KafkaApi, ConsumerGroupOffsetsResponse } from './types'; -import { DiscoveryApi } from '@backstage/core-plugin-api'; +import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api'; export class KafkaBackendClient implements KafkaApi { private readonly discoveryApi: DiscoveryApi; + private readonly identityApi: IdentityApi; - constructor(options: { discoveryApi: DiscoveryApi }) { + constructor(options: { + discoveryApi: DiscoveryApi; + identityApi: IdentityApi; + }) { this.discoveryApi = options.discoveryApi; + this.identityApi = options.identityApi; } private async internalGet(path: string): Promise { const url = `${await this.discoveryApi.getBaseUrl('kafka')}${path}`; + const idToken = await this.identityApi.getIdToken(); const response = await fetch(url, { method: 'GET', headers: { 'Content-Type': 'application/json', + ...(idToken && { Authorization: `Bearer ${idToken}` }), }, }); diff --git a/plugins/kafka/src/plugin.ts b/plugins/kafka/src/plugin.ts index e389e5e27a..539b0a0fda 100644 --- a/plugins/kafka/src/plugin.ts +++ b/plugins/kafka/src/plugin.ts @@ -21,6 +21,7 @@ import { createRoutableExtension, createRouteRef, discoveryApiRef, + identityApiRef, } from '@backstage/core-plugin-api'; export const rootCatalogKafkaRouteRef = createRouteRef({ @@ -33,8 +34,9 @@ export const kafkaPlugin = createPlugin({ apis: [ createApiFactory({ api: kafkaApiRef, - deps: { discoveryApi: discoveryApiRef }, - factory: ({ discoveryApi }) => new KafkaBackendClient({ discoveryApi }), + deps: { discoveryApi: discoveryApiRef, identityApi: identityApiRef }, + factory: ({ discoveryApi, identityApi }) => + new KafkaBackendClient({ discoveryApi, identityApi }), }), ], routes: { diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 1a7aebf2f5..3f9b77a3a3 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -16,6 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import express from 'express'; import { JsonObject } from '@backstage/types'; import { JsonValue } from '@backstage/types'; +import { Knex } from 'knex'; import { LocationSpec } from '@backstage/catalog-model'; import { Logger as Logger_2 } from 'winston'; import { Octokit } from '@octokit/rest'; @@ -55,6 +56,9 @@ export class CatalogEntityClient { ): Promise; } +// @public +export type CompletedTaskState = 'failed' | 'completed'; + // Warning: (ae-missing-release-tag) "createBuiltinActions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -189,6 +193,64 @@ export const createTemplateAction: < templateAction: TemplateAction, ) => TemplateAction; +// @public +export type CreateWorkerOptions = { + taskBroker: TaskBroker; + actionRegistry: TemplateActionRegistry; + integrations: ScmIntegrations; + workingDirectory: string; + logger: Logger_2; +}; + +// @public +export class DatabaseTaskStore implements TaskStore { + constructor(options: DatabaseTaskStoreOptions); + // (undocumented) + claimTask(): Promise; + // (undocumented) + completeTask({ + taskId, + status, + eventBody, + }: { + taskId: string; + status: Status; + eventBody: JsonObject; + }): Promise; + // Warning: (ae-forgotten-export) The symbol "DatabaseTaskStoreOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + static create(options: DatabaseTaskStoreOptions): Promise; + // (undocumented) + createTask( + spec: TaskSpec, + secrets?: TaskSecrets, + ): Promise<{ + taskId: string; + }>; + // (undocumented) + emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise; + // (undocumented) + getTask(taskId: string): Promise; + // (undocumented) + heartbeatTask(taskId: string): Promise; + // (undocumented) + listEvents({ taskId, after }: TaskStoreListEventsOptions): Promise<{ + events: SerializedTaskEvent[]; + }>; + // (undocumented) + listStaleTasks({ timeoutS }: { timeoutS: number }): Promise<{ + tasks: { + taskId: string; + }[]; + }>; +} + +// @public +export type DispatchResult = { + taskId: string; +}; + // Warning: (ae-missing-release-tag) "fetchContents" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -216,9 +278,7 @@ export class OctokitProvider { getOctokit(repoUrl: string): Promise; } -// Warning: (ae-missing-release-tag) "RouterOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export interface RouterOptions { // (undocumented) actions?: TemplateAction[]; @@ -235,6 +295,8 @@ export interface RouterOptions { // (undocumented) reader: UrlReader; // (undocumented) + taskBroker?: TaskBroker; + // (undocumented) taskWorkers?: number; } @@ -260,6 +322,216 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { validateEntityKind(entity: Entity): Promise; } +// @public +export type SerializedTask = { + id: string; + spec: TaskSpec; + status: Status; + createdAt: string; + lastHeartbeatAt?: string; + secrets?: TaskSecrets; +}; + +// @public +export type SerializedTaskEvent = { + id: number; + taskId: string; + body: JsonObject; + type: TaskEventType; + createdAt: string; +}; + +// @public +export type Status = + | 'open' + | 'processing' + | 'failed' + | 'cancelled' + | 'completed'; + +// @public +export interface TaskBroker { + // (undocumented) + claim(): Promise; + // (undocumented) + dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; + // (undocumented) + get(taskId: string): Promise; + // (undocumented) + observe( + options: { + taskId: string; + after: number | undefined; + }, + callback: ( + error: Error | undefined, + result: { + events: SerializedTaskEvent[]; + }, + ) => void, + ): { + unsubscribe: () => void; + }; + // (undocumented) + vacuumTasks(timeoutS: { timeoutS: number }): Promise; +} + +// @public +export interface TaskContext { + // (undocumented) + complete(result: CompletedTaskState, metadata?: JsonValue): Promise; + // (undocumented) + done: boolean; + // (undocumented) + emitLog(message: string, metadata?: JsonValue): Promise; + // (undocumented) + getWorkspaceName(): Promise; + // (undocumented) + secrets?: TaskSecrets; + // (undocumented) + spec: TaskSpec; +} + +// @public +export type TaskEventType = 'completion' | 'log'; + +// @public +export class TaskManager implements TaskContext { + // (undocumented) + complete(result: CompletedTaskState, metadata?: JsonObject): Promise; + // (undocumented) + static create( + state: TaskState, + storage: TaskStore, + logger: Logger_2, + ): TaskManager; + // (undocumented) + get done(): boolean; + // (undocumented) + emitLog(message: string, metadata?: JsonObject): Promise; + // (undocumented) + getWorkspaceName(): Promise; + // (undocumented) + get secrets(): TaskSecrets | undefined; + // (undocumented) + get spec(): TaskSpec; +} + +// @public +export type TaskSecrets = { + token: string | undefined; +}; + +// @public +export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3; + +// @public +export interface TaskSpecV1beta2 { + // (undocumented) + apiVersion: 'backstage.io/v1beta2'; + // (undocumented) + baseUrl?: string; + // (undocumented) + output: { + [name: string]: string; + }; + // (undocumented) + steps: Array<{ + id: string; + name: string; + action: string; + input?: JsonObject; + if?: string | boolean; + }>; + // (undocumented) + values: JsonObject; +} + +// @public +export interface TaskSpecV1beta3 { + // (undocumented) + apiVersion: 'scaffolder.backstage.io/v1beta3'; + // (undocumented) + baseUrl?: string; + // (undocumented) + output: { + [name: string]: JsonValue; + }; + // (undocumented) + parameters: JsonObject; + // Warning: (ae-forgotten-export) The symbol "TaskStep" needs to be exported by the entry point index.d.ts + // + // (undocumented) + steps: TaskStep[]; +} + +// @public +export interface TaskState { + // (undocumented) + secrets?: TaskSecrets; + // (undocumented) + spec: TaskSpec; + // (undocumented) + taskId: string; +} + +// @public +export interface TaskStore { + // (undocumented) + claimTask(): Promise; + // (undocumented) + completeTask(options: { + taskId: string; + status: Status; + eventBody: JsonObject; + }): Promise; + // (undocumented) + createTask( + task: TaskSpec, + secrets?: TaskSecrets, + ): Promise<{ + taskId: string; + }>; + // (undocumented) + emitLogEvent({ taskId, body }: TaskStoreEmitOptions): Promise; + // (undocumented) + getTask(taskId: string): Promise; + // (undocumented) + heartbeatTask(taskId: string): Promise; + // (undocumented) + listEvents({ taskId, after }: TaskStoreListEventsOptions): Promise<{ + events: SerializedTaskEvent[]; + }>; + // (undocumented) + listStaleTasks(options: { timeoutS: number }): Promise<{ + tasks: { + taskId: string; + }[]; + }>; +} + +// @public +export type TaskStoreEmitOptions = { + taskId: string; + body: JsonObject; +}; + +// @public +export type TaskStoreListEventsOptions = { + taskId: string; + after?: number | undefined; +}; + +// @public +export class TaskWorker { + // (undocumented) + static create(options: CreateWorkerOptions): Promise; + // (undocumented) + runOneTask(task: TaskContext): Promise; + // (undocumented) + start(): void; +} + // Warning: (ae-missing-release-tag) "TemplateAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) diff --git a/plugins/scaffolder-backend/sample-templates/all-templates.yaml b/plugins/scaffolder-backend/sample-templates/all-templates.yaml index 2d5adb8e3b..6637c9479b 100644 --- a/plugins/scaffolder-backend/sample-templates/all-templates.yaml +++ b/plugins/scaffolder-backend/sample-templates/all-templates.yaml @@ -6,7 +6,6 @@ metadata: spec: targets: - ./remote-templates.yaml - # For local development of a template, you can reference your local templates here. # Examples: # diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index 284e372b4c..e055223d8c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -14,3 +14,4 @@ * limitations under the License. */ export * from './actions'; +export * from './tasks'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 9c72a310c1..813d8d0f5b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -20,15 +20,15 @@ import { ConflictError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; import { - DbTaskEventRow, - DbTaskRow, + SerializedTaskEvent, + SerializedTask, Status, TaskEventType, TaskSecrets, TaskSpec, TaskStore, TaskStoreEmitOptions, - TaskStoreGetEventsOptions, + TaskStoreListEventsOptions, } from './types'; import { DateTime } from 'luxon'; @@ -54,17 +54,37 @@ export type RawDbTaskEventRow = { created_at: string; }; +/** + * DatabaseTaskStore + * + * @public + */ +export type DatabaseTaskStoreOptions = { + database: Knex; +}; + +/** + * DatabaseTaskStore + * + * @public + */ export class DatabaseTaskStore implements TaskStore { - static async create(knex: Knex): Promise { - await knex.migrate.latest({ + private readonly db: Knex; + + static async create( + options: DatabaseTaskStoreOptions, + ): Promise { + await options.database.migrate.latest({ directory: migrationsDir, }); - return new DatabaseTaskStore(knex); + return new DatabaseTaskStore(options); } - constructor(private readonly db: Knex) {} + constructor(options: DatabaseTaskStoreOptions) { + this.db = options.database; + } - async getTask(taskId: string): Promise { + async getTask(taskId: string): Promise { const [result] = await this.db('tasks') .where({ id: taskId }) .select(); @@ -101,7 +121,7 @@ export class DatabaseTaskStore implements TaskStore { return { taskId }; } - async claimTask(): Promise { + async claimTask(): Promise { return this.db.transaction(async tx => { const [task] = await tx('tasks') .where({ @@ -243,7 +263,7 @@ export class DatabaseTaskStore implements TaskStore { async listEvents({ taskId, after, - }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }> { + }: TaskStoreListEventsOptions): Promise<{ events: SerializedTaskEvent[] }> { const rawEvents = await this.db('task_events') .where({ task_id: taskId, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts index d6d1b0cfd2..d07ff29fae 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.test.ts @@ -18,16 +18,16 @@ import mockFs from 'mock-fs'; import * as winston from 'winston'; import { getVoidLogger } from '@backstage/backend-common'; -import { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; +import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner'; import { TemplateActionRegistry } from '../actions'; import { ScmIntegrations } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; -import { Task, TaskSpec } from './types'; +import { TaskContext, TaskSpec } from './types'; describe('DefaultWorkflowRunner', () => { const logger = getVoidLogger(); let actionRegistry = new TemplateActionRegistry(); - let runner: DefaultWorkflowRunner; + let runner: NunjucksWorkflowRunner; let fakeActionHandler: jest.Mock; const integrations = ScmIntegrations.fromConfig( @@ -38,7 +38,7 @@ describe('DefaultWorkflowRunner', () => { }), ); - const createMockTaskWithSpec = (spec: TaskSpec): Task => ({ + const createMockTaskWithSpec = (spec: TaskSpec): TaskContext => ({ spec, complete: async () => {}, done: false, @@ -88,7 +88,7 @@ describe('DefaultWorkflowRunner', () => { }, }); - runner = new DefaultWorkflowRunner({ + runner = new NunjucksWorkflowRunner({ actionRegistry, integrations, workingDirectory: '/tmp', diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts similarity index 98% rename from plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts rename to plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts index 6d43ae978c..77d0a7be09 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/HandlebarsWorkflowRunner.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import { - Task, + TaskContext, WorkflowRunner, WorkflowResponse, TaskSpecV1beta2, @@ -47,7 +47,7 @@ const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta2 => * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced * with the default workflow runner interface in the future so this entire thing can go bye bye. */ -export class LegacyWorkflowRunner implements WorkflowRunner { +export class HandlebarsWorkflowRunner implements WorkflowRunner { private readonly handlebars: typeof Handlebars; constructor(private readonly options: Options) { @@ -72,7 +72,7 @@ export class LegacyWorkflowRunner implements WorkflowRunner { this.handlebars.registerHelper('eq', (a, b) => a === b); } - async execute(task: Task): Promise { + async execute(task: TaskContext): Promise { if (!isValidTaskSpec(task.spec)) { throw new InputError(`Task spec is not a valid v1beta2 task spec`); } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts index 2714f1b80c..91e772dc4e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.test.ts @@ -20,12 +20,12 @@ import { createTemplateAction, TemplateActionRegistry } from '../actions'; import { ScmIntegrations } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; -import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; -import { Task, TaskSpec } from './types'; +import { HandlebarsWorkflowRunner } from './HandlebarsWorkflowRunner'; +import { TaskContext, TaskSpec } from './types'; import { RepoSpec } from '../actions/builtin/publish/util'; describe('LegacyWorkflowRunner', () => { - let runner: LegacyWorkflowRunner; + let runner: HandlebarsWorkflowRunner; const logger = getVoidLogger(); let actionRegistry = new TemplateActionRegistry(); @@ -37,7 +37,7 @@ describe('LegacyWorkflowRunner', () => { }), ); - const createMockTaskWithSpec = (spec: TaskSpec): Task => ({ + const createMockTaskWithSpec = (spec: TaskSpec): TaskContext => ({ spec, complete: async () => {}, done: false, @@ -60,7 +60,7 @@ describe('LegacyWorkflowRunner', () => { }, }); - runner = new LegacyWorkflowRunner({ + runner = new HandlebarsWorkflowRunner({ actionRegistry, integrations, workingDirectory: '/tmp', diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts similarity index 95% rename from plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts rename to plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts index 6c68e0c4c5..587a93e1f2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts @@ -15,7 +15,7 @@ */ import { ScmIntegrations } from '@backstage/integration'; import { - Task, + TaskContext, TaskSpec, TaskSpecV1beta3, TaskStep, @@ -34,7 +34,7 @@ import { validate as validateJsonSchema } from 'jsonschema'; import { parseRepoUrl } from '../actions/builtin/publish/util'; import { TemplateActionRegistry } from '../actions'; -type Options = { +type NunjucksWorkflowRunnerOptions = { workingDirectory: string; actionRegistry: TemplateActionRegistry; integrations: ScmIntegrations; @@ -52,7 +52,13 @@ const isValidTaskSpec = (taskSpec: TaskSpec): taskSpec is TaskSpecV1beta3 => { return taskSpec.apiVersion === 'scaffolder.backstage.io/v1beta3'; }; -const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { +const createStepLogger = ({ + task, + step, +}: { + task: TaskContext; + step: TaskStep; +}) => { const metadata = { stepId: step.id }; const taskLogger = winston.createLogger({ level: process.env.LOG_LEVEL || 'info', @@ -77,7 +83,7 @@ const createStepLogger = ({ task, step }: { task: Task; step: TaskStep }) => { return { taskLogger, streamLogger }; }; -export class DefaultWorkflowRunner implements WorkflowRunner { +export class NunjucksWorkflowRunner implements WorkflowRunner { private readonly nunjucks: nunjucks.Environment; private readonly nunjucksOptions: nunjucks.ConfigureOptions = { @@ -88,7 +94,7 @@ export class DefaultWorkflowRunner implements WorkflowRunner { }, }; - constructor(private readonly options: Options) { + constructor(private readonly options: NunjucksWorkflowRunnerOptions) { this.nunjucks = nunjucks.configure(this.nunjucksOptions); // TODO(blam): let's work out how we can deprecate these. @@ -162,7 +168,7 @@ export class DefaultWorkflowRunner implements WorkflowRunner { }); } - async execute(task: Task): Promise { + async execute(task: TaskContext): Promise { if (!isValidTaskSpec(task.spec)) { throw new InputError( 'Wrong template version executed with the workflow engine', diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts index 1a4d0d68c5..76fb8e00f0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -17,8 +17,8 @@ import { getVoidLogger, DatabaseManager } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { DatabaseTaskStore } from './DatabaseTaskStore'; -import { StorageTaskBroker, TaskAgent } from './StorageTaskBroker'; -import { TaskSecrets, TaskSpec, DbTaskEventRow } from './types'; +import { StorageTaskBroker, TaskManager } from './StorageTaskBroker'; +import { TaskSecrets, TaskSpec, SerializedTaskEvent } from './types'; async function createStore(): Promise { const manager = DatabaseManager.fromConfig( @@ -31,7 +31,9 @@ async function createStore(): Promise { }, }), ).forPlugin('scaffolder'); - return await DatabaseTaskStore.create(await manager.getClient()); + return await DatabaseTaskStore.create({ + database: await manager.getClient(), + }); } describe('StorageTaskBroker', () => { @@ -46,7 +48,7 @@ describe('StorageTaskBroker', () => { it('should claim a dispatched work item', async () => { const broker = new StorageTaskBroker(storage, logger); await broker.dispatch({} as TaskSpec); - await expect(broker.claim()).resolves.toEqual(expect.any(TaskAgent)); + await expect(broker.claim()).resolves.toEqual(expect.any(TaskManager)); }); it('should wait for a dispatched work item', async () => { @@ -56,7 +58,7 @@ describe('StorageTaskBroker', () => { await expect(Promise.race([promise, 'waiting'])).resolves.toBe('waiting'); await broker.dispatch({} as TaskSpec); - await expect(promise).resolves.toEqual(expect.any(TaskAgent)); + await expect(promise).resolves.toEqual(expect.any(TaskManager)); }); it('should dispatch multiple items and claim them in order', async () => { @@ -68,9 +70,9 @@ describe('StorageTaskBroker', () => { const taskA = await broker.claim(); const taskB = await broker.claim(); const taskC = await broker.claim(); - await expect(taskA).toEqual(expect.any(TaskAgent)); - await expect(taskB).toEqual(expect.any(TaskAgent)); - await expect(taskC).toEqual(expect.any(TaskAgent)); + await expect(taskA).toEqual(expect.any(TaskManager)); + await expect(taskB).toEqual(expect.any(TaskManager)); + await expect(taskC).toEqual(expect.any(TaskManager)); await expect(taskA.spec.steps[0].id).toBe('a'); await expect(taskB.spec.steps[0].id).toBe('b'); await expect(taskC.spec.steps[0].id).toBe('c'); @@ -127,8 +129,8 @@ describe('StorageTaskBroker', () => { const { taskId } = await broker1.dispatch({} as TaskSpec); - const logPromise = new Promise(resolve => { - const observedEvents = new Array(); + const logPromise = new Promise(resolve => { + const observedEvents = new Array(); broker2.observe({ taskId, after: undefined }, (_err, { events }) => { observedEvents.push(...events); diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 2a99c3eb30..7e7ebe5436 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -18,23 +18,28 @@ import { assertError } from '@backstage/errors'; import { Logger } from 'winston'; import { CompletedTaskState, - Task, + TaskContext, TaskSecrets, TaskSpec, TaskStore, TaskBroker, DispatchResult, - DbTaskEventRow, - DbTaskRow, + SerializedTaskEvent, + SerializedTask, } from './types'; -export class TaskAgent implements Task { +/** + * TaskManager + * + * @public + */ +export class TaskManager implements TaskContext { private isDone = false; private heartbeatTimeoutId?: ReturnType; static create(state: TaskState, storage: TaskStore, logger: Logger) { - const agent = new TaskAgent(state, storage, logger); + const agent = new TaskManager(state, storage, logger); agent.startTimeout(); return agent; } @@ -104,7 +109,12 @@ export class TaskAgent implements Task { } } -interface TaskState { +/** + * TaskState + * + * @public + */ +export interface TaskState { spec: TaskSpec; taskId: string; secrets?: TaskSecrets; @@ -125,11 +135,11 @@ export class StorageTaskBroker implements TaskBroker { ) {} private deferredDispatch = defer(); - async claim(): Promise { + async claim(): Promise { for (;;) { const pendingTask = await this.storage.claimTask(); if (pendingTask) { - return TaskAgent.create( + return TaskManager.create( { taskId: pendingTask.id, spec: pendingTask.spec, @@ -155,7 +165,7 @@ export class StorageTaskBroker implements TaskBroker { }; } - async get(taskId: string): Promise { + async get(taskId: string): Promise { return this.storage.getTask(taskId); } @@ -166,9 +176,9 @@ export class StorageTaskBroker implements TaskBroker { }, callback: ( error: Error | undefined, - result: { events: DbTaskEventRow[] }, + result: { events: SerializedTaskEvent[] }, ) => void, - ): () => void { + ): { unsubscribe: () => void } { const { taskId } = options; let cancelled = false; @@ -195,7 +205,7 @@ export class StorageTaskBroker implements TaskBroker { } })(); - return unsubscribe; + return { unsubscribe }; } async vacuumTasks(timeoutS: { timeoutS: number }): Promise { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts index 87a0229b5d..9b6baf3b84 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts @@ -19,8 +19,20 @@ import { ConfigReader } from '@backstage/config'; import { DatabaseTaskStore } from './DatabaseTaskStore'; import { StorageTaskBroker } from './StorageTaskBroker'; import { TaskWorker } from './TaskWorker'; -import { WorkflowRunner } from './types'; -import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; +import { HandlebarsWorkflowRunner } from './HandlebarsWorkflowRunner'; +import { ScmIntegrations } from '@backstage/integration'; +import { TemplateActionRegistry } from '../actions'; +import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner'; + +jest.mock('./HandlebarsWorkflowRunner'); +const MockedHandlebarsWorkflowRunner = + HandlebarsWorkflowRunner as jest.Mock; +MockedHandlebarsWorkflowRunner.mockImplementation(); + +jest.mock('./NunjucksWorkflowRunner'); +const MockedNunjucksWorkflowRunner = + NunjucksWorkflowRunner as jest.Mock; +MockedNunjucksWorkflowRunner.mockImplementation(); async function createStore(): Promise { const manager = DatabaseManager.fromConfig( @@ -33,18 +45,26 @@ async function createStore(): Promise { }, }), ).forPlugin('scaffolder'); - return await DatabaseTaskStore.create(await manager.getClient()); + return await DatabaseTaskStore.create({ + database: await manager.getClient(), + }); } describe('TaskWorker', () => { let storage: DatabaseTaskStore; - const workflowRunner: WorkflowRunner = { - execute: jest.fn(), - } as unknown as WorkflowRunner; - const legacyWorkflowRunner: LegacyWorkflowRunner = { + const integrations: ScmIntegrations = {} as ScmIntegrations; + + const actionRegistry: TemplateActionRegistry = {} as TemplateActionRegistry; + const workingDirectory = '/tmp/scaffolder'; + + const handlebarsWorkflowRunner: HandlebarsWorkflowRunner = { execute: jest.fn(), - } as unknown as LegacyWorkflowRunner; + } as unknown as HandlebarsWorkflowRunner; + + const workflowRunner: NunjucksWorkflowRunner = { + execute: jest.fn(), + } as unknown as NunjucksWorkflowRunner; beforeAll(async () => { storage = await createStore(); @@ -52,18 +72,22 @@ describe('TaskWorker', () => { beforeEach(() => { jest.resetAllMocks(); + MockedHandlebarsWorkflowRunner.mockImplementation( + () => handlebarsWorkflowRunner, + ); + MockedNunjucksWorkflowRunner.mockImplementation(() => workflowRunner); }); const logger = getVoidLogger(); it('should call the legacy workflow runner when the apiVersion is not beta3', async () => { const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ + const taskWorker = await TaskWorker.create({ + logger, + workingDirectory, + integrations, taskBroker: broker, - runners: { - legacyWorkflowRunner, - workflowRunner, - }, + actionRegistry, }); await broker.dispatch({ @@ -78,17 +102,23 @@ describe('TaskWorker', () => { const task = await broker.claim(); await taskWorker.runOneTask(task); - expect(legacyWorkflowRunner.execute).toHaveBeenCalled(); + expect(MockedHandlebarsWorkflowRunner).toBeCalledWith({ + actionRegistry, + integrations, + logger, + workingDirectory, + }); + expect(handlebarsWorkflowRunner.execute).toHaveBeenCalled(); }); it('should call the default workflow runner when the apiVersion is beta3', async () => { const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ + const taskWorker = await TaskWorker.create({ + logger, + workingDirectory, + integrations, taskBroker: broker, - runners: { - legacyWorkflowRunner, - workflowRunner, - }, + actionRegistry, }); await broker.dispatch({ @@ -112,12 +142,12 @@ describe('TaskWorker', () => { }); const broker = new StorageTaskBroker(storage, logger); - const taskWorker = new TaskWorker({ + const taskWorker = await TaskWorker.create({ + logger, + workingDirectory, + integrations, taskBroker: broker, - runners: { - legacyWorkflowRunner, - workflowRunner, - }, + actionRegistry, }); const { taskId } = await broker.dispatch({ diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 6a5a1f7843..8d89735361 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -14,20 +14,77 @@ * limitations under the License. */ -import { Task, TaskBroker, WorkflowRunner } from './types'; -import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; +import { TaskContext, TaskBroker, WorkflowRunner } from './types'; +import { HandlebarsWorkflowRunner } from './HandlebarsWorkflowRunner'; +import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner'; +import { Logger } from 'winston'; +import { TemplateActionRegistry } from '../actions'; +import { ScmIntegrations } from '@backstage/integration'; import { assertError } from '@backstage/errors'; -type Options = { +/** + * TaskWorkerOptions + * + * @public + */ +export type TaskWorkerOptions = { taskBroker: TaskBroker; runners: { - legacyWorkflowRunner: LegacyWorkflowRunner; + legacyWorkflowRunner: HandlebarsWorkflowRunner; workflowRunner: WorkflowRunner; }; }; +/** + * CreateWorkerOptions + * + * @public + */ +export type CreateWorkerOptions = { + taskBroker: TaskBroker; + actionRegistry: TemplateActionRegistry; + integrations: ScmIntegrations; + workingDirectory: string; + logger: Logger; +}; + +/** + * TaskWorker + * + * @public + */ export class TaskWorker { - constructor(private readonly options: Options) {} + private constructor(private readonly options: TaskWorkerOptions) {} + + static async create(options: CreateWorkerOptions): Promise { + const { + taskBroker, + logger, + actionRegistry, + integrations, + workingDirectory, + } = options; + + const legacyWorkflowRunner = new HandlebarsWorkflowRunner({ + logger, + actionRegistry, + integrations, + workingDirectory, + }); + + const workflowRunner = new NunjucksWorkflowRunner({ + actionRegistry, + integrations, + logger, + workingDirectory, + }); + + return new TaskWorker({ + taskBroker: taskBroker, + runners: { legacyWorkflowRunner, workflowRunner }, + }); + } + start() { (async () => { for (;;) { @@ -37,7 +94,7 @@ export class TaskWorker { })(); } - async runOneTask(task: Task) { + async runOneTask(task: TaskContext) { try { const { output } = task.spec.apiVersion === 'scaffolder.backstage.io/v1beta3' diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts index dd13264aed..510c862dae 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/index.ts @@ -13,7 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - export { DatabaseTaskStore } from './DatabaseTaskStore'; -export { StorageTaskBroker } from './StorageTaskBroker'; +export { TaskManager } from './StorageTaskBroker'; +export type { TaskState } from './StorageTaskBroker'; export { TaskWorker } from './TaskWorker'; +export type { CreateWorkerOptions } from './TaskWorker'; +export type { + TaskSecrets, + TaskSpec, + CompletedTaskState, + TaskStoreEmitOptions, + TaskStoreListEventsOptions, + SerializedTask, + SerializedTaskEvent, + TaskSpecV1beta2, + TaskSpecV1beta3, + Status, + TaskEventType, + TaskBroker, + TaskContext, + TaskStore, + DispatchResult, +} from './types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index b91930c56a..6cfd914bd2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -16,6 +16,11 @@ import { JsonValue, JsonObject } from '@backstage/types'; +/** + * Status + * + * @public + */ export type Status = | 'open' | 'processing' @@ -23,9 +28,19 @@ export type Status = | 'cancelled' | 'completed'; +/** + * CompletedTaskState + * + * @public + */ export type CompletedTaskState = 'failed' | 'completed'; -export type DbTaskRow = { +/** + * SerializedTask + * + * @public + */ +export type SerializedTask = { id: string; spec: TaskSpec; status: Status; @@ -34,8 +49,19 @@ export type DbTaskRow = { secrets?: TaskSecrets; }; +/** + * TaskEventType + * + * @public + */ export type TaskEventType = 'completion' | 'log'; -export type DbTaskEventRow = { + +/** + * SerializedTaskEvent + * + * @public + */ +export type SerializedTaskEvent = { id: number; taskId: string; body: JsonObject; @@ -43,6 +69,11 @@ export type DbTaskEventRow = { createdAt: string; }; +/** + * TaskSpecV1beta2 + * + * @public + */ export interface TaskSpecV1beta2 { apiVersion: 'backstage.io/v1beta2'; baseUrl?: string; @@ -64,6 +95,12 @@ export interface TaskStep { input?: JsonObject; if?: string | boolean; } + +/** + * TaskSpecV1beta3 + * + * @public + */ export interface TaskSpecV1beta3 { apiVersion: 'scaffolder.backstage.io/v1beta3'; baseUrl?: string; @@ -72,17 +109,37 @@ export interface TaskSpecV1beta3 { output: { [name: string]: JsonValue }; } +/** + * TaskSpec + * + * @public + */ export type TaskSpec = TaskSpecV1beta2 | TaskSpecV1beta3; +/** + * TaskSecrets + * + * @public + */ export type TaskSecrets = { token: string | undefined; }; +/** + * DispatchResult + * + * @public + */ export type DispatchResult = { taskId: string; }; -export interface Task { +/** + * Task + * + * @public + */ +export interface TaskContext { spec: TaskSpec; secrets?: TaskSecrets; done: boolean; @@ -91,8 +148,13 @@ export interface Task { getWorkspaceName(): Promise; } +/** + * TaskBroker + * + * @public + */ export interface TaskBroker { - claim(): Promise; + claim(): Promise; dispatch(spec: TaskSpec, secrets?: TaskSecrets): Promise; vacuumTasks(timeoutS: { timeoutS: number }): Promise; observe( @@ -102,28 +164,44 @@ export interface TaskBroker { }, callback: ( error: Error | undefined, - result: { events: DbTaskEventRow[] }, + result: { events: SerializedTaskEvent[] }, ) => void, - ): () => void; + ): { unsubscribe: () => void }; + get(taskId: string): Promise; } +/** + * TaskStoreEmitOptions + * + * @public + */ export type TaskStoreEmitOptions = { taskId: string; body: JsonObject; }; -export type TaskStoreGetEventsOptions = { +/** + * TaskStoreListEventsOptions + * + * @public + */ +export type TaskStoreListEventsOptions = { taskId: string; after?: number | undefined; }; +/** + * TaskStore + * + * @public + */ export interface TaskStore { createTask( task: TaskSpec, secrets?: TaskSecrets, ): Promise<{ taskId: string }>; - getTask(taskId: string): Promise; - claimTask(): Promise; + getTask(taskId: string): Promise; + claimTask(): Promise; completeTask(options: { taskId: string; status: Status; @@ -138,10 +216,10 @@ export interface TaskStore { listEvents({ taskId, after, - }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }>; + }: TaskStoreListEventsOptions): Promise<{ events: SerializedTaskEvent[] }>; } export type WorkflowResponse = { output: { [key: string]: JsonValue } }; export interface WorkflowRunner { - execute(task: Task): Promise; + execute(task: TaskContext): Promise; } diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index c0fdcb2ef7..1c5343d458 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -40,7 +40,13 @@ import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; -import { createRouter } from './router'; +/** + * TODO: The following should import directly from the router file. + * Due to a circular dependency between this plugin and the + * plugin-scaffolder-backend-module-cookiecutter plugin, it results in an error: + * TypeError: _pluginscaffolderbackend.createTemplateAction is not a function + */ +import { createRouter } from '../index'; const createCatalogClient = (templates: any[] = []) => ({ diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index f8ba99336f..a78b04da78 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -22,10 +22,12 @@ import { CatalogEntityClient } from '../lib/catalog'; import { validate } from 'jsonschema'; import { DatabaseTaskStore, - StorageTaskBroker, + TemplateActionRegistry, TaskWorker, -} from '../scaffolder/tasks'; -import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry'; + TemplateAction, + createBuiltinActions, +} from '../scaffolder'; +import { StorageTaskBroker } from '../scaffolder/tasks/StorageTaskBroker'; import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { ContainerRunner, @@ -38,12 +40,13 @@ import { TemplateEntityV1beta2, Entity } from '@backstage/catalog-model'; import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; import { ScmIntegrations } from '@backstage/integration'; -import { TemplateAction } from '../scaffolder/actions'; -import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; -import { LegacyWorkflowRunner } from '../scaffolder/tasks/LegacyWorkflowRunner'; -import { DefaultWorkflowRunner } from '../scaffolder/tasks/DefaultWorkflowRunner'; -import { TaskSpec } from '../scaffolder/tasks/types'; +import { TaskBroker, TaskSpec } from '../scaffolder/tasks/types'; +/** + * RouterOptions + * + * @public + */ export interface RouterOptions { logger: Logger; config: Config; @@ -53,6 +56,7 @@ export interface RouterOptions { actions?: TemplateAction[]; taskWorkers?: number; containerRunner: ContainerRunner; + taskBroker?: TaskBroker; } function isSupportedTemplate( @@ -85,34 +89,27 @@ export async function createRouter( const workingDirectory = await getWorkingDirectory(config, logger); const entityClient = new CatalogEntityClient(catalogClient); const integrations = ScmIntegrations.fromConfig(config); + let taskBroker: TaskBroker; + + if (!options.taskBroker) { + const databaseTaskStore = await DatabaseTaskStore.create({ + database: await database.getClient(), + }); + taskBroker = new StorageTaskBroker(databaseTaskStore, logger); + } else { + taskBroker = options.taskBroker; + } - const databaseTaskStore = await DatabaseTaskStore.create( - await database.getClient(), - ); - const taskBroker = new StorageTaskBroker(databaseTaskStore, logger); const actionRegistry = new TemplateActionRegistry(); - const legacyWorkflowRunner = new LegacyWorkflowRunner({ - logger, - actionRegistry, - integrations, - workingDirectory, - }); - - const workflowRunner = new DefaultWorkflowRunner({ - actionRegistry, - integrations, - logger, - workingDirectory, - }); const workers = []; for (let i = 0; i < (taskWorkers || 1); i++) { - const worker = new TaskWorker({ + const worker = await TaskWorker.create({ taskBroker, - runners: { - legacyWorkflowRunner, - workflowRunner, - }, + actionRegistry, + integrations, + logger, + workingDirectory, }); workers.push(worker); } @@ -261,7 +258,7 @@ export async function createRouter( }); // After client opens connection send all events as string - const unsubscribe = taskBroker.observe( + const { unsubscribe } = taskBroker.observe( { taskId, after }, (error, { events }) => { if (error) { diff --git a/plugins/tech-insights-backend-module-jsonfc/.eslintrc.js b/plugins/tech-insights-backend-module-jsonfc/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/tech-insights-backend-module-jsonfc/README.md b/plugins/tech-insights-backend-module-jsonfc/README.md new file mode 100644 index 0000000000..48c1e69919 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/README.md @@ -0,0 +1,85 @@ +# Tech Insights Backend JSON Rules engine fact checker module + +This is an extension to module to tech-insights-backend plugin, which provides basic framework and functionality to implement tech insights within Backstage. + +This module provides functionality to run checks against a [json-rules-engine](https://github.com/CacheControl/json-rules-engine) and provide boolean logic by simply building checks using JSON conditions. + +## Getting started + +To add this FactChecker into your Tech Insights you need to install the module into your backend application: + +```bash +# From your Backstage root directory +cd packages/backend +yarn add @backstage/plugin-tech-insights-backend-module-jsonfc +``` + +and modify the `techInsights.ts` file to contain a reference to the FactCheckers implementation. + +```diff ++import { JsonRulesEngineFactCheckerFactory } from '@backstage/plugin-tech-insights-backend-module-jsonfc'; + ++const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ ++ checks: [], ++ logger, ++}), + + const builder = new DefaultTechInsightsBuilder({ + logger, + config, + database, + discovery, + factRetrievers: [myFactRetrieverRegistration], ++ factCheckerFactory: myFactCheckerFactory + }); +``` + +By default this implementation comes with an in-memory storage to store checks. You can inject an additional data store by adding an implementation of `TechInsightCheckRegistry` into the constructor options when creating a `JsonRulesEngineFactCheckerFactory`. That can be done as follows + +```diff + const myTechInsightCheckRegistry: TechInsightCheckRegistry = // snip + const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ + checks: [], + logger, ++ checkRegistry: myTechInsightCheckRegistry + }), + +``` + +## Adding checks + +Checks for this FactChecker are constructed as [`json-rules-engine` compatible JSON rules](https://github.com/CacheControl/json-rules-engine/blob/master/docs/rules.md#conditions). A check could look like the following for example: + +```ts +import { TechInsightJsonRuleCheck } from '../types'; +import { JSON_RULE_ENGINE_CHECK_TYPE } from '../constants'; + +export const exampleCheck: TechInsightJsonRuleCheck = { + id: 'demodatacheck', // Unique identifier of this check + name: 'demodatacheck', // A human readable name of this check to be displayed in the UI + type: JSON_RULE_ENGINE_CHECK_TYPE, // Type identifier of the check. Used to run logic against, determine persistence option to use and render correct components on the UI + description: 'A fact check for demoing purposes', // A description to be displayed in the UI + factRefs: ['demo-poc.factretriever'], // References to fact containers that this check uses. See documentation on FactRetrievers for more information on these + rule: { + // The actual rule + conditions: { + all: [ + // 2 options are available, all and any conditions. + { + fact: 'examplenumberfact', // Reference to an individual fact to check against + operator: 'greaterThanInclusive', // Operator to use. See: https://github.com/CacheControl/json-rules-engine/blob/master/docs/rules.md#operators for more + value: 2, // The threshold value that the fact must satisfy + }, + ], + }, + }, + successMetadata: { + // Additional metadata to be returned if the check has passed + link: 'https://link.to.some.information.com', + }, + failureMetadata: { + // Additional metadata to be returned if the check has failed + link: 'https://sonar.mysonarqube.com/increasing-number-value', + }, +}; +``` diff --git a/plugins/tech-insights-backend-module-jsonfc/api-report.md b/plugins/tech-insights-backend-module-jsonfc/api-report.md new file mode 100644 index 0000000000..7093fbfc2b --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/api-report.md @@ -0,0 +1,114 @@ +## API Report File for "@backstage/plugin-tech-insights-backend-module-jsonfc" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BooleanCheckResult } from '@backstage/plugin-tech-insights-common'; +import { CheckResponse } from '@backstage/plugin-tech-insights-common'; +import { CheckValidationResponse } from '@backstage/plugin-tech-insights-node'; +import { FactChecker } from '@backstage/plugin-tech-insights-node'; +import { Logger as Logger_2 } from 'winston'; +import { TechInsightCheck } from '@backstage/plugin-tech-insights-node'; +import { TechInsightCheckRegistry } from '@backstage/plugin-tech-insights-node'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; +import { TopLevelCondition } from 'json-rules-engine'; + +// @public (undocumented) +export type CheckCondition = { + operator: string; + fact: string; + factValue: any; + factResult: any; + result: boolean; +}; + +// @public (undocumented) +export const JSON_RULE_ENGINE_CHECK_TYPE = 'json-rules-engine'; + +// @public (undocumented) +export interface JsonRuleBooleanCheckResult extends BooleanCheckResult { + // (undocumented) + check: JsonRuleCheckResponse; +} + +// @public (undocumented) +export interface JsonRuleCheckResponse extends CheckResponse { + // (undocumented) + rule: { + conditions: ResponseTopLevelCondition & { + priority: number; + }; + }; +} + +// @public +export class JsonRulesEngineFactChecker + implements FactChecker +{ + constructor({ + checks, + repository, + logger, + checkRegistry, + }: JsonRulesEngineFactCheckerOptions); + // (undocumented) + getChecks(): Promise; + // (undocumented) + runChecks( + entity: string, + checks?: string[], + ): Promise; + // (undocumented) + validate(check: TechInsightJsonRuleCheck): Promise; +} + +// @public +export class JsonRulesEngineFactCheckerFactory { + constructor({ + checks, + logger, + checkRegistry, + }: JsonRulesEngineFactCheckerFactoryOptions); + // (undocumented) + construct(repository: TechInsightsStore): JsonRulesEngineFactChecker; +} + +// @public +export type JsonRulesEngineFactCheckerFactoryOptions = { + checks: TechInsightJsonRuleCheck[]; + logger: Logger_2; + checkRegistry?: TechInsightCheckRegistry; +}; + +// @public +export type JsonRulesEngineFactCheckerOptions = { + checks: TechInsightJsonRuleCheck[]; + repository: TechInsightsStore; + logger: Logger_2; + checkRegistry?: TechInsightCheckRegistry; +}; + +// @public (undocumented) +export type ResponseTopLevelCondition = + | { + all: CheckCondition[]; + } + | { + any: CheckCondition[]; + }; + +// @public (undocumented) +export type Rule = { + conditions: TopLevelCondition; + name?: string; + priority?: number; +}; + +// @public (undocumented) +export interface TechInsightJsonRuleCheck extends TechInsightCheck { + // (undocumented) + rule: Rule; +} + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/tech-insights-backend-module-jsonfc/package.json b/plugins/tech-insights-backend-module-jsonfc/package.json new file mode 100644 index 0000000000..8943b0e461 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/package.json @@ -0,0 +1,52 @@ +{ + "name": "@backstage/plugin-tech-insights-backend-module-jsonfc", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/tech-insights-backend-module-jsonfc" + }, + "keywords": [ + "backstage", + "tech-insights", + "scorecard" + ], + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/backend-common": "^0.9.0", + "@backstage/config": "^0.1.8", + "@backstage/errors": "^0.1.1", + "@backstage/plugin-tech-insights-common": "^0.1.0", + "@backstage/plugin-tech-insights-node": "^0.1.0", + "ajv": "^7.0.3", + "json-rules-engine": "^6.1.2", + "lodash": "^4.17.21", + "luxon": "^2.0.2", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/cli": "^0.8.0", + "@types/node-cron": "^2.0.4" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/tech-insights-backend-module-jsonfc/src/constants.ts b/plugins/tech-insights-backend-module-jsonfc/src/constants.ts new file mode 100644 index 0000000000..5fd69a3fef --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/constants.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @public + */ +export const JSON_RULE_ENGINE_CHECK_TYPE = 'json-rules-engine'; diff --git a/plugins/tech-insights-backend-module-jsonfc/src/index.ts b/plugins/tech-insights-backend-module-jsonfc/src/index.ts new file mode 100644 index 0000000000..1239211f73 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/index.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { + JsonRulesEngineFactCheckerFactory, + JsonRulesEngineFactChecker, +} from './service/JsonRulesEngineFactChecker'; +export { JSON_RULE_ENGINE_CHECK_TYPE } from './constants'; +export type { + JsonRulesEngineFactCheckerFactoryOptions, + JsonRulesEngineFactCheckerOptions, +} from './service/JsonRulesEngineFactChecker'; +export type { + JsonRuleCheckResponse, + JsonRuleBooleanCheckResult, + TechInsightJsonRuleCheck, + ResponseTopLevelCondition, + Rule, + CheckCondition, +} from './types'; diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts new file mode 100644 index 0000000000..dec84a8634 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/CheckRegistry.ts @@ -0,0 +1,60 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConflictError, NotFoundError } from '@backstage/errors'; +import { + TechInsightCheck, + TechInsightCheckRegistry, +} from '@backstage/plugin-tech-insights-node'; + +export class DefaultCheckRegistry + implements TechInsightCheckRegistry +{ + private readonly checks = new Map(); + + constructor(checks: CheckType[]) { + checks.forEach(check => { + this.register(check); + }); + } + + async register(check: CheckType): Promise { + if (this.checks.has(check.id)) { + throw new ConflictError( + `Tech insight check with id ${check.id} has already been registered`, + ); + } + this.checks.set(check.id, check); + return check; + } + + async get(checkId: string): Promise { + const check = this.checks.get(checkId); + if (!check) { + throw new NotFoundError( + `Tech insight check with id '${checkId}' is not registered.`, + ); + } + return check; + } + async getAll(checks: string[]): Promise { + return Promise.all(checks.map(checkId => this.get(checkId))); + } + + async list(): Promise { + return [...this.checks.values()]; + } +} diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts new file mode 100644 index 0000000000..77b3e6c8b1 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.test.ts @@ -0,0 +1,301 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + TechInsightCheckRegistry, + TechInsightsStore, +} from '@backstage/plugin-tech-insights-node'; +import { + JSON_RULE_ENGINE_CHECK_TYPE, + JsonRulesEngineFactCheckerFactory, +} from '../index'; +import { getVoidLogger } from '@backstage/backend-common'; +import { TechInsightJsonRuleCheck } from '../types'; + +const testChecks: Record = { + broken: [ + { + id: 'brokenTestCheck', + name: 'brokenTestCheck', + type: JSON_RULE_ENGINE_CHECK_TYPE, + description: 'Broken Check For Testing', + factIds: ['test-factretriever'], + rule: { + conditions: { + all: [ + { + fact: 'testnumberfact', + operator: 'largerThan', + value: 1, + }, + ], + }, + }, + }, + ], + broken2: [ + { + id: 'brokenTestCheck2', + name: 'brokenTestCheck2', + type: JSON_RULE_ENGINE_CHECK_TYPE, + description: 'Second Broken Check For Testing', + factIds: ['non-existing-factretriever'], + rule: { + conditions: { + any: [ + { + fact: 'somefact', + operator: 'lessThan', + value: 1, + }, + ], + }, + }, + }, + ], + simple: [ + { + id: 'simpleTestCheck', + name: 'simpleTestCheck', + type: JSON_RULE_ENGINE_CHECK_TYPE, + description: 'Simple Check For Testing', + factIds: ['test-factretriever'], + rule: { + conditions: { + all: [ + { + fact: 'testnumberfact', + operator: 'lessThan', + value: 5, + }, + ], + }, + }, + }, + ], + + simple2: [ + { + id: 'simpleTestCheck2', + name: 'simpleTestCheck2', + type: JSON_RULE_ENGINE_CHECK_TYPE, + description: 'Second Simple Check For Testing', + factIds: ['test-factretriever'], + rule: { + conditions: { + all: [ + { + fact: 'testnumberfact', + operator: 'greaterThan', + value: 2, + }, + ], + }, + }, + }, + ], +}; + +const latestSchemasMock = jest.fn().mockImplementation(() => [ + { + version: '0.0.1', + id: 2, + ref: 'test-factretriever', + entityTypes: ['component'], + testnumberfact: { + type: 'integer', + description: '', + }, + }, +]); +const factsBetweenTimestampsByIdsMock = jest.fn(); +const latestFactsByIdsMock = jest.fn().mockImplementation(() => ({})); +const mockCheckRegistry = { + getAll(checks: string[]) { + return checks.flatMap(check => testChecks[check]); + }, +} as unknown as TechInsightCheckRegistry; + +const mockRepository: TechInsightsStore = { + getLatestFactsByIds: latestFactsByIdsMock, + getFactsBetweenTimestampsByIds: factsBetweenTimestampsByIdsMock, + getLatestSchemas: latestSchemasMock, +} as unknown as TechInsightsStore; + +describe('JsonRulesEngineFactChecker', () => { + const factChecker = new JsonRulesEngineFactCheckerFactory({ + checkRegistry: mockCheckRegistry, + checks: [], + logger: getVoidLogger(), + }).construct(mockRepository); + + describe('when running checks', () => { + it('should throw on incorrectly configured checks conditions', async () => { + const cur = async () => await factChecker.runChecks('a/a/a', ['broken']); + await expect(cur()).rejects.toThrowError( + 'Failed to run rules engine, Unknown operator: largerThan', + ); + }); + + it('should handle cases where wrong facts are referenced', async () => { + const cur = async () => await factChecker.runChecks('a/a/a', ['broken2']); + await expect(cur()).rejects.toThrowError( + 'Failed to run rules engine, Undefined fact: somefact', + ); + }); + it('should respond with result, facts, fact schemas and checks', async () => { + latestFactsByIdsMock.mockImplementation(() => + Promise.resolve({ + ['test-factretriever']: { + id: 'test-factretriever', + facts: { + testnumberfact: 3, + }, + }, + }), + ); + const results = await factChecker.runChecks('a/a/a', ['simple']); + expect(results).toHaveLength(1); + expect(results[0]).toMatchObject({ + facts: { + testnumberfact: { + value: 3, + type: 'integer', + description: '', + }, + }, + result: true, + check: { + id: 'simpleTestCheck', + type: JSON_RULE_ENGINE_CHECK_TYPE, + name: 'simpleTestCheck', + description: 'Simple Check For Testing', + factIds: ['test-factretriever'], + rule: { + conditions: { + all: [ + { + fact: 'testnumberfact', + factResult: 3, + operator: 'lessThan', + result: true, + value: 5, + }, + ], + priority: 1, + }, + }, + }, + }); + }); + + it('should gracefully handle multiple check at once', async () => { + latestFactsByIdsMock.mockImplementation(() => + Promise.resolve({ + ['test-factretriever']: { + id: 'test-factretriever', + facts: { + testnumberfact: 3, + }, + }, + }), + ); + const results = await factChecker.runChecks('a/a/a', [ + 'simple', + 'simple2', + ]); + expect(results).toMatchObject([ + { + facts: { + testnumberfact: { + value: 3, + type: 'integer', + description: '', + }, + }, + result: true, + check: { + id: 'simpleTestCheck', + type: JSON_RULE_ENGINE_CHECK_TYPE, + name: 'simpleTestCheck', + description: 'Simple Check For Testing', + factIds: ['test-factretriever'], + rule: { + conditions: { + priority: 1, + all: [ + { + operator: 'lessThan', + value: 5, + fact: 'testnumberfact', + factResult: 3, + result: true, + }, + ], + }, + }, + }, + }, + { + facts: { + testnumberfact: { + value: 3, + type: 'integer', + description: '', + }, + }, + result: true, + check: { + id: 'simpleTestCheck2', + type: JSON_RULE_ENGINE_CHECK_TYPE, + name: 'simpleTestCheck2', + description: 'Second Simple Check For Testing', + factIds: ['test-factretriever'], + rule: { + conditions: { + priority: 1, + all: [ + { + operator: 'greaterThan', + value: 2, + fact: 'testnumberfact', + factResult: 3, + result: true, + }, + ], + }, + }, + }, + }, + ]); + }); + }); + + describe('when validating checks', () => { + it('should succeed on valid rules', async () => { + const validationResponse = await factChecker.validate( + testChecks.simple[0], + ); + expect(validationResponse.valid).toBeTruthy(); + }); + it('should fail on broken rules', async () => { + const validationResponse = await factChecker.validate( + testChecks.broken[0], + ); + expect(validationResponse.valid).toBeFalsy(); + }); + }); +}); diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts new file mode 100644 index 0000000000..ae1f18a373 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts @@ -0,0 +1,343 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonRuleBooleanCheckResult, TechInsightJsonRuleCheck } from '../types'; +import { + FactChecker, + TechInsightCheckRegistry, + FlatTechInsightFact, + TechInsightsStore, + CheckValidationResponse, +} from '@backstage/plugin-tech-insights-node'; +import { FactResponse } from '@backstage/plugin-tech-insights-common'; +import { Engine, EngineResult, TopLevelCondition } from 'json-rules-engine'; +import { DefaultCheckRegistry } from './CheckRegistry'; +import { Logger } from 'winston'; +import { pick } from 'lodash'; +import Ajv from 'ajv'; +import * as validationSchema from './validation-schema.json'; +import { JSON_RULE_ENGINE_CHECK_TYPE } from '../constants'; + +const noopEvent = { + type: 'noop', +}; + +/** + * @public + * Should actually be at-internal + * + * Constructor options for JsonRulesEngineFactChecker + */ +export type JsonRulesEngineFactCheckerOptions = { + checks: TechInsightJsonRuleCheck[]; + repository: TechInsightsStore; + logger: Logger; + checkRegistry?: TechInsightCheckRegistry; +}; + +/** + * @public + * Should actually be at-internal + * + * FactChecker implementation using json-rules-engine + */ +export class JsonRulesEngineFactChecker + implements FactChecker +{ + private readonly checkRegistry: TechInsightCheckRegistry; + private repository: TechInsightsStore; + private readonly logger: Logger; + + constructor({ + checks, + repository, + logger, + checkRegistry, + }: JsonRulesEngineFactCheckerOptions) { + this.repository = repository; + this.logger = logger; + checks.forEach(check => this.validate(check)); + this.checkRegistry = + checkRegistry ?? + new DefaultCheckRegistry(checks); + } + + async runChecks( + entity: string, + checks?: string[], + ): Promise { + const engine = new Engine(); + const techInsightChecks = checks + ? await this.checkRegistry.getAll(checks) + : await this.checkRegistry.list(); + const factIds = techInsightChecks.flatMap(it => it.factIds); + const facts = await this.repository.getLatestFactsByIds(factIds, entity); + techInsightChecks.forEach(techInsightCheck => { + const rule = techInsightCheck.rule; + rule.name = techInsightCheck.id; + engine.addRule({ ...techInsightCheck.rule, event: noopEvent }); + }); + const factValues = Object.values(facts).reduce( + (acc, it) => ({ ...acc, ...it.facts }), + {}, + ); + + try { + const results = await engine.run(factValues); + return await this.ruleEngineResultsToCheckResponse( + results, + techInsightChecks, + Object.values(facts), + ); + } catch (e) { + throw new Error(`Failed to run rules engine, ${e.message}`); + } + } + + async validate( + check: TechInsightJsonRuleCheck, + ): Promise { + const ajv = new Ajv({ verbose: true }); + const validator = ajv.compile(validationSchema); + const isValidToSchema = validator(check.rule); + if (check.type !== JSON_RULE_ENGINE_CHECK_TYPE) { + const msg = `Only ${JSON_RULE_ENGINE_CHECK_TYPE} checks can be registered to this fact checker`; + this.logger.warn(msg); + return { + valid: false, + message: msg, + }; + } + if (!isValidToSchema) { + const msg = 'Failed to to validate conditions against JSON schema'; + this.logger.warn( + 'Failed to to validate conditions against JSON schema', + validator.errors, + ); + return { + valid: false, + message: msg, + errors: validator.errors ? validator.errors : undefined, + }; + } + + const existingSchemas = await this.repository.getLatestSchemas( + check.factIds, + ); + const references = this.retrieveIndividualFactReferences( + check.rule.conditions, + ); + const results = references.map(ref => ({ + ref, + result: existingSchemas.some(schema => schema.hasOwnProperty(ref)), + })); + const failedReferences = results.filter(it => !it.result); + failedReferences.forEach(it => { + this.logger.warn( + `Validation failed for check ${check.name}. Reference to value ${ + it.ref + } does not exists in referred fact schemas: ${check.factIds.join(',')}`, + ); + }); + const valid = failedReferences.length === 0; + return { + valid, + ...(!valid + ? { + message: `Check is referencing missing values from fact schemas: ${failedReferences + .map(it => it.ref) + .join(',')}`, + } + : {}), + }; + } + + getChecks(): Promise { + return this.checkRegistry.list(); + } + + private retrieveIndividualFactReferences( + condition: TopLevelCondition | { fact: string }, + ): string[] { + let results: string[] = []; + if ('all' in condition) { + results = results.concat( + condition.all.flatMap(con => + this.retrieveIndividualFactReferences(con), + ), + ); + } else if ('any' in condition) { + results = results.concat( + condition.any.flatMap(con => + this.retrieveIndividualFactReferences(con), + ), + ); + } else { + results.push(condition.fact); + } + return results; + } + + private async ruleEngineResultsToCheckResponse( + results: EngineResult, + techInsightChecks: TechInsightJsonRuleCheck[], + facts: FlatTechInsightFact[], + ) { + return await Promise.all( + [ + ...(results.results && results.results), + ...(results.failureResults && results.failureResults), + ].map(async result => { + const techInsightCheck = techInsightChecks.find( + check => check.id === result.name, + ); + if (!techInsightCheck) { + // This should never happen, we just constructed these based on each other + throw new Error( + `Failed to determine tech insight check with id ${result.name}. Discrepancy between ran rule engine and configured checks.`, + ); + } + const factResponse = await this.constructFactInformationResponse( + facts, + techInsightCheck, + ); + return { + facts: factResponse, + result: result.result, + check: JsonRulesEngineFactChecker.constructCheckResponse( + techInsightCheck, + result, + ), + }; + }), + ); + } + + private static constructCheckResponse( + techInsightCheck: TechInsightJsonRuleCheck, + result: any, + ) { + const returnable = { + id: techInsightCheck.id, + type: techInsightCheck.type, + name: techInsightCheck.name, + description: techInsightCheck.description, + factIds: techInsightCheck.factIds, + metadata: result.result + ? techInsightCheck.successMetadata + : techInsightCheck.failureMetadata, + rule: { conditions: {} }, + }; + + if ('toJSON' in result) { + // Results from json-rules-engine serialize "wrong" since the objects are creating their own serialization implementations. + // 'toJSON' should always be present in the result object but it is missing from the types. + // Parsing the stringified representation into a plain object here to be able to serialize it later + // along with other items present in the returned response. + const rule = JSON.parse(result.toJSON()); + return { ...returnable, rule: pick(rule, ['conditions']) }; + } + return returnable; + } + + private async constructFactInformationResponse( + facts: FlatTechInsightFact[], + techInsightCheck: TechInsightJsonRuleCheck, + ): Promise { + const factSchemas = await this.repository.getLatestSchemas( + techInsightCheck.factIds, + ); + const schemas = factSchemas.reduce( + (acc, schema) => ({ ...acc, ...schema }), + {}, + ); + const individualFacts = this.retrieveIndividualFactReferences( + techInsightCheck.rule.conditions, + ); + const factValues = facts + .filter(factContainer => + techInsightCheck.factIds.includes(factContainer.id), + ) + .reduce( + (acc, factContainer) => ({ + ...acc, + ...pick(factContainer.facts, individualFacts), + }), + {}, + ); + return Object.entries(factValues).reduce((acc, [key, value]) => { + return { + ...acc, + [key]: { + value, + ...schemas[key], + }, + }; + }, {}); + } +} + +/** + * @public + * + * Constructor options for JsonRulesEngineFactCheckerFactory + * + * Implementation of checkRegistry is optional. + * If there is a need to use persistent storage for checks, it is recommended to inject a storage implementation here. + * Otherwise an in-memory option is instantiated and used. + */ +export type JsonRulesEngineFactCheckerFactoryOptions = { + checks: TechInsightJsonRuleCheck[]; + logger: Logger; + checkRegistry?: TechInsightCheckRegistry; +}; + +/** + * @public + * + * Factory to construct JsonRulesEngineFactChecker + * Can be constructed with optional implementation of CheckInsightCheckRegistry if needed. + * Otherwise defaults to using in-memory CheckRegistry + */ +export class JsonRulesEngineFactCheckerFactory { + private readonly checks: TechInsightJsonRuleCheck[]; + private readonly logger: Logger; + private readonly checkRegistry?: TechInsightCheckRegistry; + + constructor({ + checks, + logger, + checkRegistry, + }: JsonRulesEngineFactCheckerFactoryOptions) { + this.logger = logger; + this.checks = checks; + this.checkRegistry = checkRegistry; + } + + /** + * @param repository - Implementation of TechInsightsStore. Used by the returned JsonRulesEngineFactChecker + * to retrieve fact and fact schema data + * @returns JsonRulesEngineFactChecker implementation + */ + construct(repository: TechInsightsStore) { + return new JsonRulesEngineFactChecker({ + checks: this.checks, + logger: this.logger, + checkRegistry: this.checkRegistry, + repository, + }); + } +} diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/validation-schema.json b/plugins/tech-insights-backend-module-jsonfc/src/service/validation-schema.json new file mode 100644 index 0000000000..9020ba0735 --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/validation-schema.json @@ -0,0 +1,208 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://github.com/CacheControl/json-rules-engine/rule-schema.json", + "type": "object", + "title": "Fact Checks", + "description": "Checks contain a set of conditions and a single event. When the engine is run, each check condition is evaluated and results returned", + "required": ["conditions"], + "properties": { + "conditions": { + "$ref": "#/definitions/conditions" + }, + "priority": { + "$id": "#/properties/priority", + "anyOf": [ + { + "type": "integer", + "minimum": 1 + } + ], + "title": "Priority", + "description": "Dictates when check should be run, relative to other check. Higher priority checks are run before lower priority checks. Checks with the same priority are run in parallel. Priority must be a positive, non-zero integer.", + "default": 1, + "examples": [1] + } + }, + "definitions": { + "conditions": { + "type": "object", + "title": "Conditions", + "description": "Check conditions are a combination of facts, operators, and values that determine whether the check is a success or a failure. The simplest form of a condition consists of a fact, an operator, and a value. When the engine runs, the operator is used to compare the fact against the value. Each check's conditions must have either an all or an any operator at its root, containing an array of conditions. The all operator specifies that all conditions contained within must be truthy for the check to be considered a success. The any operator only requires one condition to be truthy for the check to succeed.", + "default": {}, + "examples": [ + { + "all": [ + { + "value": true, + "fact": "displayMessage", + "operator": "equal" + } + ] + } + ], + "oneOf": [ + { + "required": ["any"] + }, + { + "required": ["all"] + } + ], + "properties": { + "any": { + "$ref": "#/definitions/conditionArray" + }, + "all": { + "$ref": "#/definitions/conditionArray" + } + } + }, + "conditionArray": { + "type": "array", + "title": "Condition Array", + "description": "An array of conditions with a possible recursive inclusion of another condition array.", + "default": [], + "items": { + "anyOf": [ + { + "$ref": "#/definitions/conditions" + }, + { + "$ref": "#/definitions/condition" + } + ] + } + }, + "condition": { + "type": "object", + "title": "Condition", + "description": "Check conditions are a combination of facts, operators, and values that determine whether the check is a success or a failure. The simplest form of a condition consists of a fact, an operator, and a value. When the engine runs, the operator is used to compare the fact against the value.", + "default": { + "fact": "my-fact", + "operator": "lessThanInclusive", + "value": 1 + }, + "examples": [ + { + "fact": "gameDuration", + "operator": "equal", + "value": 40.0 + }, + { + "value": 5.0, + "fact": "personalFoulCount", + "operator": "greaterThanInclusive" + }, + { + "fact": "product-price", + "operator": "greaterThan", + "path": "$.price", + "value": 100.0, + "params": { + "productId": "widget" + } + } + ], + "required": ["fact", "operator", "value"], + "properties": { + "fact": { + "type": "string", + "title": "Fact", + "description": "Facts are methods or constants registered with the engine prior to runtime and referenced within check conditions. Each fact method should be a pure function that may return a either computed value, or promise that resolves to a computed value. As check conditions are evaluated during runtime, they retrieve fact values dynamically and use the condition operator to compare the fact result with the condition value.", + "default": "", + "examples": ["gameDuration"] + }, + "operator": { + "type": "string", + "anyOf": [ + { + "const": "equal", + "title": "fact must equal value" + }, + { + "const": "notEqual", + "title": "fact must not equal value" + }, + { + "const": "lessThan", + "title": "fact must be less than value" + }, + { + "const": "lessThanInclusive", + "title": "fact must be less than or equal to value" + }, + { + "const": "greaterThan", + "title": "fact must be greater than value" + }, + { + "const": "greaterThanInclusive", + "title": "fact must be greater than or equal to value" + }, + { + "const": "in", + "title": "fact must be included in value (an array)" + }, + { + "const": "notIn", + "title": "fact must not be included in value (an array)" + }, + { + "const": "contains", + "title": "fact (an array) must include value" + }, + { + "const": "doesNotContain", + "title": "fact (an array) must not include value" + } + ], + "title": "Operator", + "description": "The operator compares the value returned by the fact to what is stored in the value property. If the result is truthy, the condition passes.", + "default": "", + "examples": ["equal"] + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object" + }, + { + "type": "array" + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ], + "title": "Value", + "description": "The value the fact should be compared to.", + "default": 0, + "examples": [40] + }, + "params": { + "type": "object", + "title": "Event Params", + "description": "Optional helper params to make available to the event processor.", + "default": {}, + "examples": [ + { + "customProperty": "customValue" + } + ] + }, + "path": { + "type": "string", + "title": "Path", + "description": "For more complex data structures, writing a separate fact handler for each object property quickly becomes verbose and unwieldy. To address this, a path property may be provided to traverse fact data using json-path syntax. Json-path support is provided by jsonpath-plus", + "default": "", + "examples": ["$.price"] + } + } + } + } +} diff --git a/plugins/tech-insights-backend-module-jsonfc/src/setupTests.ts b/plugins/tech-insights-backend-module-jsonfc/src/setupTests.ts new file mode 100644 index 0000000000..a330613afb --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/plugins/tech-insights-backend-module-jsonfc/src/types.ts b/plugins/tech-insights-backend-module-jsonfc/src/types.ts new file mode 100644 index 0000000000..592fee248d --- /dev/null +++ b/plugins/tech-insights-backend-module-jsonfc/src/types.ts @@ -0,0 +1,73 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { TopLevelCondition } from 'json-rules-engine'; +import { TechInsightCheck } from '@backstage/plugin-tech-insights-node'; +import { + BooleanCheckResult, + CheckResponse, +} from '@backstage/plugin-tech-insights-common'; + +/** + * @public + */ +export type Rule = { + conditions: TopLevelCondition; + name?: string; + priority?: number; +}; + +/** + * @public + */ +export interface TechInsightJsonRuleCheck extends TechInsightCheck { + rule: Rule; +} + +/** + * @public + */ +export type CheckCondition = { + operator: string; + fact: string; + factValue: any; + factResult: any; + result: boolean; +}; + +/** + * @public + */ +export type ResponseTopLevelCondition = + | { all: CheckCondition[] } + | { any: CheckCondition[] }; + +/** + * @public + */ +export interface JsonRuleCheckResponse extends CheckResponse { + rule: { + conditions: ResponseTopLevelCondition & { + priority: number; + }; + }; +} + +/** + * @public + */ +export interface JsonRuleBooleanCheckResult extends BooleanCheckResult { + check: JsonRuleCheckResponse; +} diff --git a/plugins/tech-insights-backend/.eslintrc.js b/plugins/tech-insights-backend/.eslintrc.js new file mode 100644 index 0000000000..a126c438a4 --- /dev/null +++ b/plugins/tech-insights-backend/.eslintrc.js @@ -0,0 +1,11 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], + rules: { + 'jest/expect-expect': [ + 'error', + { + assertFunctionNames: ['expect', 'request.**.expect'], + }, + ], + }, +}; diff --git a/plugins/tech-insights-backend/README.md b/plugins/tech-insights-backend/README.md new file mode 100644 index 0000000000..e2cc1c0252 --- /dev/null +++ b/plugins/tech-insights-backend/README.md @@ -0,0 +1,230 @@ +# Tech Insights Backend + +This is the backend for the default Backstage Tech Insights feature. +This provides the API for the frontend tech insights, scorecards and fact visualization functionality, +as well as a framework to run fact retrievers and store fact values in to a data store. + +## Installation + +### Install the package + +```bash +# From your Backstage root directory +cd packages/backend +yarn add @backstage/plugin-tech-insights-backend +``` + +### Adding the plugin to your `packages/backend` + +You'll need to add the plugin to the router in your `backend` package. You can +do this by creating a file called `packages/backend/src/plugins/techInsights.ts`. An example content for `techInsights.ts` could be something like this. + +```ts +import { + createRouter, + DefaultTechInsightsBuilder, +} from '@backstage/plugin-tech-insights-backend'; +import { Router } from 'express'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin({ + logger, + config, + discovery, + database, +}: PluginEnvironment): Promise { + const builder = new DefaultTechInsightsBuilder({ + logger, + config, + database, + discovery, + factRetrievers: [], // Fact retrievers registrations you want tech insights to use + }); + + return await createRouter({ + ...(await builder.build()), + logger, + config, + }); +} +``` + +With the `techInsights.ts` router setup in place, add the router to +`packages/backend/src/index.ts`: + +```diff ++import techInsights from './plugins/techInsights'; + + async function main() { + ... + const createEnv = makeCreateEnv(config); + + const catalogEnv = useHotMemoize(module, () => createEnv('catalog')); ++ const techInsightsEnv = useHotMemoize(module, () => createEnv('tech_insights')); + + const apiRouter = Router(); ++ apiRouter.use('/tech-insights', await techInsights(techInsightsEnv)); + ... + apiRouter.use(notFoundHandler()); + } +``` + +### Adding fact retrievers + +At this point the Tech Insights backend is installed in your backend package, but +you will not have any fact retrievers present in your application. To have the implemented FactRetrieverEngine within this package to be able to retrieve and store fact data into the database, you need to add these. + +To create factRetrieverRegistration you need to implement `FactRetriever` interface defined in `@backstage/plugin-tech-insights-common` package. After you have implemented this interface you can wrap that into a registration object like follows: + +```ts +import { createFactRetrieverRegistration } from './createFactRetriever'; + +const myFactRetriever = { + /** + * snip + */ +}; + +const myFactRetrieverRegistration = createFactRetrieverRegistration( + '1 * 3 * * ', // On the first minute of the third day of the month + myFactRetriever, +); +``` + +Then you can modify the example `techInsights.ts` file shown above like this: + +```diff +const builder = new DefaultTechInsightsBuilder({ + logger, + config, + database, + discovery, +- factRetrievers: [], ++ factRetrievers: [myFactRetrieverRegistration], +}); +``` + +#### Running fact retrievers in a multi-instance installation + +Current logic on running scheduled fact retrievers is intended to be executed in a single instance. Running on multi-instane environment there might be some additional data accumulation when multiple fact retrievers would retrieve and persist their facts. To mitigate this it is recommended to mark a single instance to be a specific fact retriever instance. One way to do this is by using environment variables to indicate if the retrievers should be registered. This can be done for example like the code snippet below + +```diff +const builder = new DefaultTechInsightsBuilder({ + logger, + config, + database, + discovery, +- factRetrievers: [], ++ factRetrievers: process.env.MAIN_FACT_RETRIEVER_INSTANCE ? [myFactRetrieverRegistration] : [], +}); +``` + +Where the instance dedicated to handling retrieval of facts would have environment variable `MAIN_FACT_RETRIEVER_INSTANCE` set to true. + +### Creating Fact Retrievers + +A Fact Retriever consist of four required and one optional parts: + +1. `id` - unique identifier of a fact retriever +2. `version`: A semver string indicating the current version of the schema and the handler +3. `schema` - A versioned schema defining the shape of data a fact retriever returns +4. `handler` - An asynchronous function handling the logic of retrieving and returning facts for an entity +5. `entityFilter` - (Optional) EntityFilter object defining the entity kinds, types and/or names this fact retriever handles + +An example implementation of a FactRetriever could for example be as follows: + +```ts +const myFactRetriever: FactRetriever = { + id: 'documentation-number-factretriever', // unique identifier of the fact retriever + version: '0.1.1', // SemVer version number of this fact retriever schema. This should be incremented if the implementation changes + entityFilter: [{ kind: 'component' }], // EntityFilter to be used in the future (creating checks, graphs etc.) to figure out which entities this fact retrieves data for. + schema: { + // Name/identifier of an individual fact that this retriever returns + examplenumberfact: { + type: 'integer', // Type of the fact + description: 'A fact of a number', // Description of the fact + }, + }, + handler: async ctx => { + // Handler function that retrieves the fact + const { discovery, config, logger } = ctx; + const catalogClient = new CatalogClient({ + discoveryApi: discovery, + }); + const entities = await catalogClient.getEntities({ + filter: [{ kind: 'component' }], + }); + /** + * snip: Do complex logic to retrieve facts from external system or calculate fact values + */ + + // Respond with an array of entity/fact values + return entities.items.map(it => { + return { + // Entity information that this fact relates to + entity: { + namespace: it.metadata.namespace, + kind: it.kind, + name: it.metadata.name, + }, + + // All facts that this retriever returns + facts: { + examplenumberfact: 2, // + }, + // (optional) timestamp to use as a Luxon DateTime object + }; + }); + }, +}; +``` + +### Adding a fact checker + +This module comes with a possibility to additionally add a fact checker and expose fact checking endpoints from the API. To be able to enable this feature you need to add a FactCheckerFactory implementation to be part of the `DefaultTechInsightsBuilder` constructor call. + +There is a default FactChecker implementation provided in module `@backstage/plugin-tech-insights-backend-module-jsonfc`. This implementation uses `json-rules-engine` as the underlying functionality to run checks. If you want to implement your own FactChecker, for example to be able to handle other than `boolean` result types, you can do so by implementing `FactCheckerFactory` and `FactChecker` interfaces from `@backstage/plugin-tech-insights-common` package. + +To add the default FactChecker into your Tech Insights you need to install the module into your backend application: + +```bash +# From your Backstage root directory +cd packages/backend +yarn add @backstage/plugin-tech-insights-backend-module-jsonfc +``` + +and modify the `techInsights.ts` file to contain a reference to the FactChecker implementation. + +```diff ++import { JsonRulesEngineFactCheckerFactory } from '@backstage/plugin-tech-insights-backend-module-jsonfc'; + ++const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ ++ checks: [], ++ logger, ++}), + + const builder = new DefaultTechInsightsBuilder({ + logger, + config, + database, + discovery, + factRetrievers: [myFactRetrieverRegistration], ++ factCheckerFactory: myFactCheckerFactory + }); +``` + +To be able to run checks, you need to additionally add individual checks into your FactChecker implementation. For examples how to add these, you can check the documentation of the individual implementation of the FactChecker + +#### Modifying check persistence + +The default FactChecker implementation comes with an in-memory storage to store checks. You can inject an additional data store by adding an implementation of `TechInsightCheckRegistry` into the constructor options when creating a `JsonRulesEngineFactCheckerFactory`. That can be done as follows: + +```diff +const myTechInsightCheckRegistry: TechInsightCheckRegistry = // snip +const myFactCheckerFactory = new JsonRulesEngineFactCheckerFactory({ + checks: [], + logger, ++ checkRegistry: myTechInsightCheckRegistry +}), + +``` diff --git a/plugins/tech-insights-backend/api-report.md b/plugins/tech-insights-backend/api-report.md new file mode 100644 index 0000000000..826789dd1d --- /dev/null +++ b/plugins/tech-insights-backend/api-report.md @@ -0,0 +1,82 @@ +## API Report File for "@backstage/plugin-tech-insights-backend" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { CheckResult } from '@backstage/plugin-tech-insights-common'; +import { Config } from '@backstage/config'; +import express from 'express'; +import { FactChecker } from '@backstage/plugin-tech-insights-node'; +import { FactCheckerFactory } from '@backstage/plugin-tech-insights-node'; +import { FactRetriever } from '@backstage/plugin-tech-insights-node'; +import { FactRetrieverRegistration } from '@backstage/plugin-tech-insights-node'; +import { Logger as Logger_2 } from 'winston'; +import { PluginDatabaseManager } from '@backstage/backend-common'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { TechInsightCheck } from '@backstage/plugin-tech-insights-node'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; + +// @public +export const buildTechInsightsContext: < + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +>( + options: TechInsightsOptions, +) => Promise>; + +// @public +export function createFactRetrieverRegistration( + cadence: string, + factRetriever: FactRetriever, +): FactRetrieverRegistration; + +// @public +export function createRouter< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +>(options: RouterOptions): Promise; + +// @public +export type PersistenceContext = { + techInsightsStore: TechInsightsStore; +}; + +// @public +export interface RouterOptions< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + config: Config; + factChecker?: FactChecker; + logger: Logger_2; + persistenceContext: PersistenceContext; +} + +// @public (undocumented) +export type TechInsightsContext< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> = { + factChecker?: FactChecker; + persistenceContext: PersistenceContext; +}; + +// @public (undocumented) +export interface TechInsightsOptions< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + // (undocumented) + config: Config; + // (undocumented) + database: PluginDatabaseManager; + // (undocumented) + discovery: PluginEndpointDiscovery; + factCheckerFactory?: FactCheckerFactory; + factRetrievers: FactRetrieverRegistration[]; + // (undocumented) + logger: Logger_2; +} + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js b/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js new file mode 100644 index 0000000000..6094daa4b7 --- /dev/null +++ b/plugins/tech-insights-backend/migrations/202109061111_fact_schemas.js @@ -0,0 +1,60 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.createTable('fact_schemas', table => { + table.comment( + 'The table for tech insight fact schemas. Containing a versioned data model definition for a collection of facts.', + ); + table + .text('id') + .notNullable() + .comment('Identifier of the fact retriever plugin/package'); + table + .string('version') + .notNullable() + .comment('SemVer string defining the version of schema.'); + table + .string('entityFilter') + .nullable() + .comment( + 'A serialized entity filter object used to determine which entities this schema is applicable to.', + ); + table + .text('schema') + .notNullable() + .comment( + 'Fact schema defining the values/types what this version of the fact would contain.', + ); + table.primary(['id', 'version']); + table.index('id', 'fact_schema_id_idx'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('fact_schemas', table => { + table.dropIndex([], 'fact_schema_id_idx'); + }); + await knex.schema.dropTable('fact_schemas'); +}; diff --git a/plugins/tech-insights-backend/migrations/202109061212_facts.js b/plugins/tech-insights-backend/migrations/202109061212_facts.js new file mode 100644 index 0000000000..0bdcdc0da1 --- /dev/null +++ b/plugins/tech-insights-backend/migrations/202109061212_facts.js @@ -0,0 +1,72 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.createTable('facts', table => { + table.comment( + 'The table for tech insight fact collections. Contains facts for individual fact retriever namespace/ref.', + ); + table + .text('id') + .notNullable() + .comment('Unique identifier of the fact retriever plugin/package'); + table + .string('version') + .notNullable() + .comment( + 'SemVer string defining the version of schema this fact is based on.', + ); + table + .dateTime('timestamp') + .defaultTo(knex.fn.now()) + .notNullable() + .comment('The timestamp when this entry was created'); + table + .text('entity') + .notNullable() + .comment('Identifier of the entity these facts relate to'); + table + .text('facts') + .notNullable() + .comment( + 'Values of the fact collection stored as key-value pairs in JSON format.', + ); + + table + .foreign(['id', 'version']) + .references(['id', 'version']) + .inTable('fact_schemas'); + + table.index(['id', 'entity'], 'fact_id_entity_idx'); + table.index('id', 'fact_id_idx'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('facts', table => { + table.dropIndex([], 'fact_id_idx'); + table.dropIndex([], 'fact_id_entity_idx'); + }); + await knex.schema.dropTable('facts'); +}; diff --git a/plugins/tech-insights-backend/package.json b/plugins/tech-insights-backend/package.json new file mode 100644 index 0000000000..678bec25c4 --- /dev/null +++ b/plugins/tech-insights-backend/package.json @@ -0,0 +1,66 @@ +{ + "name": "@backstage/plugin-tech-insights-backend", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/tech-insights-backend" + }, + "keywords": [ + "backstage", + "tech-insights", + "reporting" + ], + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/backend-common": "^0.9.0", + "@backstage/catalog-client": "^0.3.16", + "@backstage/catalog-model": "^0.9.0", + "@backstage/config": "^0.1.8", + "@backstage/errors": "^0.1.1", + "@backstage/plugin-tech-insights-common": "^0.1.0", + "@backstage/plugin-tech-insights-node": "^0.1.0", + "@types/express": "^4.17.6", + "cross-fetch": "^3.0.6", + "express": "^4.17.1", + "express-promise-router": "^4.1.0", + "knex": "^0.95.1", + "lodash": "^4.17.21", + "luxon": "^2.0.2", + "node-cron": "^3.0.0", + "semver": "^7.3.5", + "uuid": "^8.3.2", + "winston": "^3.2.1", + "yn": "^4.0.0" + }, + "devDependencies": { + "@backstage/backend-test-utils": "^0.1.8", + "@backstage/cli": "^0.8.0", + "@types/supertest": "^2.0.8", + "@types/node-cron": "^3.0.0", + "@types/semver": "^7.3.8", + "supertest": "^6.1.3" + }, + "files": [ + "dist", + "migrations/**/*.{js,d.ts}" + ] +} diff --git a/plugins/tech-insights-backend/src/index.ts b/plugins/tech-insights-backend/src/index.ts new file mode 100644 index 0000000000..5e8e690e8d --- /dev/null +++ b/plugins/tech-insights-backend/src/index.ts @@ -0,0 +1,27 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './service/router'; +export type { RouterOptions } from './service/router'; + +export { buildTechInsightsContext } from './service/techInsightsContextBuilder'; +export type { + TechInsightsOptions, + TechInsightsContext, +} from './service/techInsightsContextBuilder'; + +export type { PersistenceContext } from './service/persistence/persistenceContext'; +export { createFactRetrieverRegistration } from './service/fact/createFactRetriever'; diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts new file mode 100644 index 0000000000..4dc002465b --- /dev/null +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.test.ts @@ -0,0 +1,149 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + FactRetriever, + FactRetrieverRegistration, + FactSchemaDefinition, + TechInsightFact, + TechInsightsStore, +} from '@backstage/plugin-tech-insights-node'; +import { FactRetrieverRegistry } from './FactRetrieverRegistry'; +import { FactRetrieverEngine } from './FactRetrieverEngine'; +import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { schedule } from 'node-cron'; + +jest.mock('node-cron', () => { + const original = jest.requireActual('node-cron'); + return { + ...original, + schedule: jest.fn(), + }; +}); + +const testFactRetriever: FactRetriever = { + id: 'test-factretriever', + version: '0.0.1', + entityFilter: [{ kind: 'component' }], + schema: { + testnumberfact: { + type: 'integer', + description: '', + }, + }, + handler: async () => { + return [ + { + entity: { + namespace: 'a', + kind: 'a', + name: 'a', + }, + facts: { + testnumberfact: 1, + }, + }, + ]; + }, +}; +const cadence = '1 * * * *'; +describe('FactRetrieverEngine', () => { + let engine: FactRetrieverEngine; + let factSchemaAssertionCallback: ( + factSchemaDefinition: FactSchemaDefinition, + ) => void; + let factInsertionAssertionCallback: (facts: TechInsightFact[]) => void; + + const mockRepository: TechInsightsStore = { + insertFacts: (facts: TechInsightFact[]) => { + factInsertionAssertionCallback(facts); + return Promise.resolve(); + }, + insertFactSchema: (def: FactSchemaDefinition) => { + factSchemaAssertionCallback(def); + return Promise.resolve(); + }, + } as unknown as TechInsightsStore; + + const mockFactRetrieverRegistry: FactRetrieverRegistry = { + listRetrievers(): FactRetriever[] { + return [testFactRetriever]; + }, + listRegistrations(): FactRetrieverRegistration[] { + return [{ factRetriever: testFactRetriever, cadence }]; + }, + } as unknown as FactRetrieverRegistry; + + const defaultEngineConfig = { + factRetrieverContext: { + logger: getVoidLogger(), + config: ConfigReader.fromConfigs([]), + discovery: { + getBaseUrl: (_: string) => Promise.resolve('http://mock.url'), + getExternalBaseUrl: (_: string) => Promise.resolve('http://mock.url'), + }, + }, + factRetrieverRegistry: mockFactRetrieverRegistry, + repository: mockRepository, + }; + + it('Should update fact retriever schemas on initialization', async () => { + factSchemaAssertionCallback = ({ id, schema, version, entityFilter }) => { + expect(id).toEqual('test-factretriever'); + expect(version).toEqual('0.0.1'); + expect(entityFilter).toEqual([{ kind: 'component' }]); + expect(schema).toEqual({ + testnumberfact: { + type: 'integer', + description: '', + }, + }); + }; + + engine = await FactRetrieverEngine.create(defaultEngineConfig); + }); + it('Should insert facts when scheduled step is run', async () => { + (schedule as jest.Mock).mockImplementation( + (cronCadence: string, retrieverAction: Function) => { + return { + cadence: cronCadence, + triggerScheduledJobNow: retrieverAction, + }; + }, + ); + + factSchemaAssertionCallback = () => {}; + factInsertionAssertionCallback = facts => { + expect(facts).toHaveLength(1); + expect(facts[0]).toEqual({ + ref: 'test-factretriever', + entity: { + namespace: 'a', + kind: 'a', + name: 'a', + }, + facts: { + testnumberfact: 1, + }, + }); + }; + engine = await FactRetrieverEngine.create(defaultEngineConfig); + engine.schedule(); + const job: any = engine.getJob('test-factretriever'); + job.triggerScheduledJobNow(); + expect(job.cadence!!).toEqual(cadence); + }); +}); diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts new file mode 100644 index 0000000000..aa2207cca8 --- /dev/null +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverEngine.ts @@ -0,0 +1,135 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + FactRetriever, + FactRetrieverContext, + TechInsightsStore, +} from '@backstage/plugin-tech-insights-node'; +import { FactRetrieverRegistry } from './FactRetrieverRegistry'; +import { schedule, validate, ScheduledTask } from 'node-cron'; +import { Logger } from 'winston'; + +function randomDailyCron() { + const rand = (min: number, max: number) => + Math.floor(Math.random() * (max - min + 1) + min); + return `${rand(0, 59)} ${rand(0, 23)} * * *`; +} + +function duration(startTimestamp: [number, number]): string { + const delta = process.hrtime(startTimestamp); + const seconds = delta[0] + delta[1] / 1e9; + return `${seconds.toFixed(1)}s`; +} + +export class FactRetrieverEngine { + private scheduledJobs = new Map(); + + constructor( + private readonly repository: TechInsightsStore, + private readonly factRetrieverRegistry: FactRetrieverRegistry, + private readonly factRetrieverContext: FactRetrieverContext, + private readonly logger: Logger, + private readonly defaultCadence?: string, + ) {} + + static async create({ + repository, + factRetrieverRegistry, + factRetrieverContext, + defaultCadence, + }: { + repository: TechInsightsStore; + factRetrieverRegistry: FactRetrieverRegistry; + factRetrieverContext: FactRetrieverContext; + defaultCadence?: string; + }) { + await Promise.all( + factRetrieverRegistry + .listRetrievers() + .map(it => repository.insertFactSchema(it)), + ); + + return new FactRetrieverEngine( + repository, + factRetrieverRegistry, + factRetrieverContext, + factRetrieverContext.logger, + defaultCadence, + ); + } + + schedule() { + const registrations = this.factRetrieverRegistry.listRegistrations(); + const newRegs: string[] = []; + registrations.forEach(registration => { + const { factRetriever, cadence } = registration; + if (!this.scheduledJobs.has(factRetriever.id)) { + const cronExpression = + cadence || this.defaultCadence || randomDailyCron(); + if (!validate(cronExpression)) { + this.logger.warn( + `Validation failed for cron expression ${cronExpression} when trying to schedule fact retriever ${factRetriever.id}`, + ); + return; + } + const job = schedule( + cronExpression, + this.createFactRetrieverHandler(factRetriever), + ); + this.scheduledJobs.set(factRetriever.id, job); + newRegs.push(factRetriever.id); + } + }); + this.logger.info( + `Scheduled ${newRegs.length} fact retrievers to Fact Retriever Engine.`, + ); + } + + getJob(ref: string) { + return this.scheduledJobs.get(ref); + } + + private createFactRetrieverHandler(factRetriever: FactRetriever) { + return async () => { + const startTimestamp = process.hrtime(); + this.logger.info( + `Retrieving facts for fact retriever ${factRetriever.id}`, + ); + const facts = await factRetriever.handler(this.factRetrieverContext); + if (this.logger.isDebugEnabled()) { + this.logger.debug( + `Retrieved ${facts.length} facts for fact retriever ${ + factRetriever.id + } in ${duration(startTimestamp)}`, + ); + } + + try { + await this.repository.insertFacts(factRetriever.id, facts); + this.logger.info( + `Stored ${facts.length} facts for fact retriever ${ + factRetriever.id + } in ${duration(startTimestamp)}`, + ); + } catch (e) { + this.logger.warn( + `Failed to insert facts for fact retriever ${factRetriever.id}`, + e, + ); + } + }; + } +} diff --git a/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts b/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts new file mode 100644 index 0000000000..719f8bf504 --- /dev/null +++ b/plugins/tech-insights-backend/src/service/fact/FactRetrieverRegistry.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + FactRetriever, + FactRetrieverRegistration, + FactSchema, +} from '@backstage/plugin-tech-insights-node'; +import { ConflictError, NotFoundError } from '@backstage/errors'; + +export class FactRetrieverRegistry { + private readonly retrievers = new Map(); + + constructor(retrievers: FactRetrieverRegistration[]) { + retrievers.forEach(it => { + this.register(it); + }); + } + + register(registration: FactRetrieverRegistration) { + if (this.retrievers.has(registration.factRetriever.id)) { + throw new ConflictError( + `Tech insight fact retriever with identifier '${registration.factRetriever.id}' has already been registered`, + ); + } + this.retrievers.set(registration.factRetriever.id, registration); + } + + get(retrieverReference: string): FactRetriever { + const registration = this.retrievers.get(retrieverReference); + if (!registration) { + throw new NotFoundError( + `Tech insight fact retriever with identifier '${retrieverReference}' is not registered.`, + ); + } + return registration.factRetriever; + } + + listRetrievers(): FactRetriever[] { + return [...this.retrievers.values()].map(it => it.factRetriever); + } + + listRegistrations(): FactRetrieverRegistration[] { + return [...this.retrievers.values()]; + } + + getSchemas(): FactSchema[] { + return this.listRetrievers().map(it => it.schema); + } +} diff --git a/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts b/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts new file mode 100644 index 0000000000..655736400a --- /dev/null +++ b/plugins/tech-insights-backend/src/service/fact/createFactRetriever.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + FactRetriever, + FactRetrieverRegistration, +} from '@backstage/plugin-tech-insights-node'; + +/** + * @public + * + * A helper function to construct fact retriever registrations. + * + * @param cadence - cron expression to indicate when the fact retriever should be triggered + * @param factRetriever - Implementation of fact retriever consisting of at least id, version, schema and handler + * + * + * @remarks + * + * Cron expressions help: + * ┌────────────── second (optional) + # │ ┌──────────── minute + # │ │ ┌────────── hour + # │ │ │ ┌──────── day of month + # │ │ │ │ ┌────── month + # │ │ │ │ │ ┌──── day of week + # │ │ │ │ │ │ + # │ │ │ │ │ │ + # * * * * * * + * + */ +export function createFactRetrieverRegistration( + cadence: string, + factRetriever: FactRetriever, +): FactRetrieverRegistration { + return { + cadence, + factRetriever, + }; +} diff --git a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts new file mode 100644 index 0000000000..43d00e2425 --- /dev/null +++ b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.test.ts @@ -0,0 +1,266 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { DateTime, Duration } from 'luxon'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; +import { Knex } from 'knex'; +import { TestDatabases } from '@backstage/backend-test-utils'; +import { getVoidLogger } from '@backstage/backend-common'; +import { initializePersistenceContext } from './persistenceContext'; + +const factSchemas = [ + { + id: 'test-fact', + version: '0.0.1-test', + entityFilter: JSON.stringify([{ kind: 'component' }]), + schema: JSON.stringify({ + testNumberFact: { + type: 'integer', + description: 'Test fact with a number type', + }, + }), + }, +]; +const additionalFactSchemas = [ + { + id: 'test-fact', + version: '1.2.1-test', + entityFilter: JSON.stringify([{ kind: 'component' }]), + schema: JSON.stringify({ + testNumberFact: { + type: 'integer', + description: 'Test fact with a number type', + }, + testStringFact: { + type: 'string', + description: 'Test fact with a string type', + }, + }), + }, + { + id: 'test-fact', + version: '1.1.1-test', + entityFilter: JSON.stringify([{ kind: 'component' }]), + schema: JSON.stringify({ + testStringFact: { + type: 'string', + description: 'Test fact with a string type', + }, + }), + }, +]; + +const secondSchema = { + id: 'second-test-fact', + version: '0.0.1-test', + entityFilter: JSON.stringify([{ kind: 'service' }]), + schema: JSON.stringify({ + testStringFact: { + type: 'string', + description: 'Test fact with a string type', + }, + }), +}; + +const now = DateTime.now().toISO(); +const shortlyInTheFuture = DateTime.now() + .plus(Duration.fromMillis(555)) + .toISO(); +const farInTheFuture = DateTime.now() + .plus(Duration.fromMillis(555666777)) + .toISO(); + +const facts = [ + { + timestamp: now, + id: 'test-fact', + version: '0.0.1-test', + entity: 'a:a/a', + facts: JSON.stringify({ + testNumberFact: 1, + }), + }, + { + timestamp: shortlyInTheFuture, + id: 'test-fact', + version: '0.0.1-test', + entity: 'a:a/a', + facts: JSON.stringify({ + testNumberFact: 2, + }), + }, +]; + +const additionalFacts = [ + { + timestamp: farInTheFuture, + id: 'test-fact', + version: '0.0.1-test', + entity: 'a:a/a', + facts: JSON.stringify({ + testNumberFact: 3, + }), + }, +]; + +describe('Tech Insights database', () => { + let store: TechInsightsStore; + let testDbClient: Knex; + beforeAll(async () => { + testDbClient = await TestDatabases.create().init('SQLITE_3'); + + store = ( + await initializePersistenceContext(testDbClient, { + logger: getVoidLogger(), + }) + ).techInsightsStore; + + await testDbClient.batchInsert('fact_schemas', factSchemas); + await testDbClient.batchInsert('facts', facts); + }); + + const baseAssertionFact = { + id: 'test-fact', + entity: { namespace: 'a', kind: 'a', name: 'a' }, + timestamp: DateTime.fromISO(shortlyInTheFuture), + version: '0.0.1-test', + facts: { testNumberFact: 2 }, + }; + + it('should be able to return latest schema', async () => { + const schemas = await store.getLatestSchemas(); + expect(schemas[0]).toMatchObject({ + id: 'test-fact', + version: '0.0.1-test', + entityFilter: [{ kind: 'component' }], + testNumberFact: { + type: 'integer', + description: 'Test fact with a number type', + }, + }); + }); + + it('should return last schema based on semver', async () => { + await testDbClient.batchInsert('fact_schemas', additionalFactSchemas); + + const schemas = await store.getLatestSchemas(); + expect(schemas[0]).toMatchObject({ + id: 'test-fact', + version: '1.2.1-test', + entityFilter: [{ kind: 'component' }], + testNumberFact: { + type: 'integer', + description: 'Test fact with a number type', + }, + testStringFact: { + type: 'string', + description: 'Test fact with a string type', + }, + }); + }); + + it('should return multiple schemas if those exists', async () => { + await testDbClient.batchInsert('fact_schemas', [ + { + ...secondSchema, + id: 'second', + }, + ]); + + const schemas = await store.getLatestSchemas(); + expect(schemas).toHaveLength(2); + expect(schemas[0]).toMatchObject({ + id: 'test-fact', + version: '1.2.1-test', + entityFilter: [{ kind: 'component' }], + testNumberFact: { + type: 'integer', + description: 'Test fact with a number type', + }, + testStringFact: { + type: 'string', + description: 'Test fact with a string type', + }, + }); + expect(schemas[1]).toMatchObject({ + id: 'second', + version: '0.0.1-test', + entityFilter: [{ kind: 'service' }], + testStringFact: { + type: 'string', + description: 'Test fact with a string type', + }, + }); + }); + + it('should return latest facts only for the correct id', async () => { + const returnedFact = await store.getLatestFactsByIds( + ['test-fact'], + 'a:a/a', + ); + expect(returnedFact['test-fact']).toMatchObject(baseAssertionFact); + }); + + it('should return latest facts for multiple ids', async () => { + await testDbClient.batchInsert('fact_schemas', [secondSchema]); + await testDbClient.batchInsert( + 'facts', + additionalFacts.map(fact => ({ + ...fact, + id: 'second-test-fact', + timestamp: farInTheFuture, + })), + ); + const returnedFacts = await store.getLatestFactsByIds( + ['test-fact', 'second-test-fact'], + 'a:a/a', + ); + + expect(returnedFacts['test-fact']).toMatchObject({ + ...baseAssertionFact, + }); + expect(returnedFacts['second-test-fact']).toMatchObject({ + ...baseAssertionFact, + id: 'second-test-fact', + timestamp: DateTime.fromISO(farInTheFuture), + facts: { testNumberFact: 3 }, + }); + }); + + it('should return facts correctly between time range', async () => { + await testDbClient.batchInsert('facts', additionalFacts); + const returnedFacts = await store.getFactsBetweenTimestampsByIds( + ['test-fact'], + 'a:a/a', + DateTime.fromISO(now), + DateTime.fromISO(shortlyInTheFuture).plus(Duration.fromMillis(10)), + ); + expect(returnedFacts['test-fact']).toHaveLength(2); + + expect(returnedFacts['test-fact'][0]).toMatchObject({ + ...baseAssertionFact, + timestamp: DateTime.fromISO(now), + facts: { testNumberFact: 1 }, + }); + expect(returnedFacts['test-fact'][1]).toMatchObject({ + ...baseAssertionFact, + }); + expect(returnedFacts['test-fact']).not.toContainEqual({ + ...baseAssertionFact, + timestamp: DateTime.fromISO(farInTheFuture), + facts: { testNumberFact: 3 }, + }); + }); +}); diff --git a/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts new file mode 100644 index 0000000000..31a2997af5 --- /dev/null +++ b/plugins/tech-insights-backend/src/service/persistence/TechInsightsDatabase.ts @@ -0,0 +1,190 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Knex } from 'knex'; +import { + FactSchema, + TechInsightFact, + FlatTechInsightFact, + TechInsightsStore, + FactSchemaDefinition, +} from '@backstage/plugin-tech-insights-node'; +import { rsort } from 'semver'; +import { groupBy, omit } from 'lodash'; +import { DateTime } from 'luxon'; +import { Logger } from 'winston'; +import { parseEntityName, stringifyEntityRef } from '@backstage/catalog-model'; + +export type RawDbFactRow = { + id: string; + version: string; + timestamp: Date | string; + entity: string; + facts: string; +}; + +type RawDbFactSchemaRow = { + id: string; + version: string; + schema: string; + entityFilter?: string; +}; + +export class TechInsightsDatabase implements TechInsightsStore { + private readonly CHUNK_SIZE = 50; + + constructor(private readonly db: Knex, private readonly logger: Logger) {} + + async getLatestSchemas(ids?: string[]): Promise { + const queryBuilder = this.db('fact_schemas'); + if (ids) { + queryBuilder.whereIn('id', ids); + } + const existingSchemas = await queryBuilder.orderBy('id', 'desc').select(); + + const groupedSchemas = groupBy(existingSchemas, 'id'); + return Object.values(groupedSchemas) + .map(schemas => { + const sorted = rsort(schemas.map(it => it.version)); + return schemas.find(it => it.version === sorted[0])!!; + }) + .map((it: RawDbFactSchemaRow) => ({ + ...omit(it, 'schema'), + ...JSON.parse(it.schema), + entityFilter: it.entityFilter ? JSON.parse(it.entityFilter) : null, + })); + } + + async insertFactSchema(schemaDefinition: FactSchemaDefinition) { + const { id, version, schema, entityFilter } = schemaDefinition; + const existingSchemas = await this.db('fact_schemas') + .where({ id }) + .and.where({ version }) + .select(); + + if (!existingSchemas || existingSchemas.length === 0) { + await this.db('fact_schemas').insert({ + id, + version, + entityFilter: entityFilter ? JSON.stringify(entityFilter) : undefined, + schema: JSON.stringify(schema), + }); + } + } + + async insertFacts(id: string, facts: TechInsightFact[]): Promise { + if (facts.length === 0) return; + const currentSchema = await this.getLatestSchema(id); + const factRows = facts.map(it => { + return { + id, + version: currentSchema.version, + entity: stringifyEntityRef(it.entity), + facts: JSON.stringify(it.facts), + ...(it.timestamp && { timestamp: it.timestamp.toJSDate() }), + }; + }); + await this.db.transaction(async tx => { + await tx.batchInsert('facts', factRows, this.CHUNK_SIZE); + }); + } + + async getLatestFactsByIds( + ids: string[], + entityTriplet: string, + ): Promise<{ [factId: string]: FlatTechInsightFact }> { + const results = await this.db('facts') + .where({ entity: entityTriplet }) + .and.whereIn('id', ids) + .join( + this.db('facts') + .max('timestamp') + .column('id as subId') + .groupBy('id') + .as('subQ'), + 'facts.id', + 'subQ.subId', + ); + return this.dbFactRowsToTechInsightFacts(results); + } + + async getFactsBetweenTimestampsByIds( + ids: string[], + entityTriplet: string, + startDateTime: DateTime, + endDateTime: DateTime, + ): Promise<{ + [factId: string]: FlatTechInsightFact[]; + }> { + const results = await this.db('facts') + .where({ entity: entityTriplet }) + .and.whereIn('id', ids) + .and.whereBetween('timestamp', [ + startDateTime.toISO(), + endDateTime.toISO(), + ]); + + return groupBy( + results.map(it => { + const { namespace, kind, name } = parseEntityName(it.entity); + const timestamp = + typeof it.timestamp === 'string' + ? DateTime.fromISO(it.timestamp) + : DateTime.fromJSDate(it.timestamp); + return { + id: it.id, + entity: { namespace, kind, name }, + timestamp, + version: it.version, + facts: JSON.parse(it.facts), + }; + }), + 'id', + ); + } + + private async getLatestSchema(id: string): Promise { + const existingSchemas = await this.db('fact_schemas') + .where({ id }) + .orderBy('id', 'desc') + .select(); + if (existingSchemas.length < 1) { + this.logger.warn(`No schema found for ${id}. `); + throw new Error(`No schema found for ${id}. `); + } + const sorted = rsort(existingSchemas.map(it => it.version)); + return existingSchemas.find(it => it.version === sorted[0])!!; + } + + private dbFactRowsToTechInsightFacts(rows: RawDbFactRow[]) { + return rows.reduce((acc, it) => { + const { namespace, kind, name } = parseEntityName(it.entity); + const timestamp = + typeof it.timestamp === 'string' + ? DateTime.fromISO(it.timestamp) + : DateTime.fromJSDate(it.timestamp); + return { + ...acc, + [it.id]: { + id: it.id, + entity: { namespace, kind, name }, + timestamp, + version: it.version, + facts: JSON.parse(it.facts), + }, + }; + }, {}); + } +} diff --git a/plugins/tech-insights-backend/src/service/persistence/persistenceContext.ts b/plugins/tech-insights-backend/src/service/persistence/persistenceContext.ts new file mode 100644 index 0000000000..b0fb65349f --- /dev/null +++ b/plugins/tech-insights-backend/src/service/persistence/persistenceContext.ts @@ -0,0 +1,59 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common'; +import { Knex } from 'knex'; +import { Logger } from 'winston'; +import { TechInsightsDatabase } from './TechInsightsDatabase'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-tech-insights-backend', + 'migrations', +); + +/** + * A Container for persistence related components in TechInsights + * + * @public + */ +export type PersistenceContext = { + techInsightsStore: TechInsightsStore; +}; + +export type CreateDatabaseOptions = { + logger: Logger; +}; + +const defaultOptions: CreateDatabaseOptions = { + logger: getVoidLogger(), +}; + +/** + * A factory method to construct persistence context for running implementation. + * + * @public + */ +export const initializePersistenceContext = async ( + knex: Knex, + options: CreateDatabaseOptions = defaultOptions, +): Promise => { + await knex.migrate.latest({ + directory: migrationsDir, + }); + return { + techInsightsStore: new TechInsightsDatabase(knex, options.logger), + }; +}; diff --git a/plugins/tech-insights-backend/src/service/router.test.ts b/plugins/tech-insights-backend/src/service/router.test.ts new file mode 100644 index 0000000000..0b7d3b7c45 --- /dev/null +++ b/plugins/tech-insights-backend/src/service/router.test.ts @@ -0,0 +1,130 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { buildTechInsightsContext } from './techInsightsContextBuilder'; +import { createRouter } from './router'; +import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import request from 'supertest'; +import express from 'express'; +import { PersistenceContext } from './persistence/persistenceContext'; +import { TechInsightsStore } from '@backstage/plugin-tech-insights-node'; +import { DateTime } from 'luxon'; +import { Knex } from 'knex'; + +describe('Tech Insights router tests', () => { + let app: express.Express; + + const latestFactsByIdsMock = jest.fn(); + const factsBetweenTimestampsByIdsMock = jest.fn(); + const latestSchemasMock = jest.fn(); + + const mockPersistenceContext: PersistenceContext = { + techInsightsStore: { + getLatestFactsByIds: latestFactsByIdsMock, + getFactsBetweenTimestampsByIds: factsBetweenTimestampsByIdsMock, + getLatestSchemas: latestSchemasMock, + } as unknown as TechInsightsStore, + }; + + afterEach(() => { + jest.resetAllMocks(); + }); + + beforeAll(async () => { + const techInsightsContext = await buildTechInsightsContext({ + database: { + getClient: () => { + return Promise.resolve({ + migrate: { + latest: () => {}, + }, + }) as unknown as Promise; + }, + }, + logger: getVoidLogger(), + factRetrievers: [], + config: ConfigReader.fromConfigs([]), + discovery: { + getBaseUrl: (_: string) => Promise.resolve('http://mock.url'), + getExternalBaseUrl: (_: string) => Promise.resolve('http://mock.url'), + }, + }); + + const router = await createRouter({ + logger: getVoidLogger(), + config: ConfigReader.fromConfigs([]), + ...techInsightsContext, + persistenceContext: mockPersistenceContext, + }); + + app = express().use(router); + }); + + it('should be able to retrieve latest schemas', async () => { + await request(app).get('/fact-schemas').expect(200); + expect(latestSchemasMock).toHaveBeenCalled(); + }); + + it('should not contain check endpoints when checker not present', async () => { + await request(app).get('/checks').expect(404); + await request(app).post('/checks/a/a/a').expect(404); + }); + + it('should be able to parse id request params for fact retrieval', async () => { + await request(app) + .get('/facts/latest') + .query({ + entity: 'a:a/a', + ids: ['firstId', 'secondId'], + }) + .expect(200); + expect(latestFactsByIdsMock).toHaveBeenCalledWith( + ['firstId', 'secondId'], + 'a:a/a', + ); + }); + + it('should be able to parse datetime request params for fact retrieval', async () => { + await request(app) + .get('/facts/range') + .query({ + entity: 'a:a/a', + ids: ['firstId', 'secondId'], + startDatetime: '2021-12-12T12:12:12', + endDatetime: '2022-11-11T11:11:11', + }) + .expect(200); + expect(factsBetweenTimestampsByIdsMock).toHaveBeenCalledWith( + ['firstId', 'secondId'], + 'a:a/a', + DateTime.fromISO('2021-12-12T12:12:12.000+00:00'), + DateTime.fromISO('2022-11-11T11:11:11.000+00:00'), + ); + }); + + it('should respond gracefully on parsing errors', async () => { + await request(app) + .get('/facts/range') + .query({ + entity: 'a:a/a', + ids: ['firstId', 'secondId'], + startDatetime: '2021-12-1222T12:12:12', + endDatetime: '2022-1122-11T11:11:11', + }) + .expect(422); + expect(latestFactsByIdsMock).toHaveBeenCalledTimes(0); + }); +}); diff --git a/plugins/tech-insights-backend/src/service/router.ts b/plugins/tech-insights-backend/src/service/router.ts new file mode 100644 index 0000000000..a8951318ea --- /dev/null +++ b/plugins/tech-insights-backend/src/service/router.ts @@ -0,0 +1,162 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import express from 'express'; +import Router from 'express-promise-router'; +import { Config } from '@backstage/config'; +import { + FactChecker, + TechInsightCheck, +} from '@backstage/plugin-tech-insights-node'; + +import { CheckResult } from '@backstage/plugin-tech-insights-common'; +import { Logger } from 'winston'; +import { DateTime } from 'luxon'; +import { PersistenceContext } from './persistence/persistenceContext'; +import { + EntityRef, + parseEntityName, + stringifyEntityRef, +} from '@backstage/catalog-model'; + +/** + * @public + * + * RouterOptions to construct TechInsights endpoints + * @typeParam CheckType - Type of the check for the fact checker this builder returns + * @typeParam CheckResultType - Type of the check result for the fact checker this builder returns + */ +export interface RouterOptions< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + /** + * Optional FactChecker implementation. If omitted, endpoints are not constructed + */ + factChecker?: FactChecker; + + /** + * TechInsights PersistenceContext. Should contain an implementation of TechInsightsStore + */ + persistenceContext: PersistenceContext; + + /** + * Backstage config object + */ + config: Config; + + /** + * Implementation of Winston logger + */ + logger: Logger; +} + +/** + * @public + * + * Constructs a tech-insights router. + * + * Exposes endpoints to handle facts + * Exposes optional endpoints to handle checks if a FactChecker implementation is passed in + * + * @param options - RouterOptions object + */ +export async function createRouter< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +>(options: RouterOptions): Promise { + const router = Router(); + router.use(express.json()); + const { persistenceContext, factChecker, logger } = options; + const { techInsightsStore } = persistenceContext; + + if (factChecker) { + logger.info('Fact checker configured. Enabling fact checking endpoints.'); + router.get('/checks', async (_req, res) => { + return res.send(await factChecker.getChecks()); + }); + + router.post('/checks/run/:namespace/:kind/:name', async (req, res) => { + const { namespace, kind, name } = req.params; + try { + if (!('checks' in req.body)) { + return res.status(422).send({ + message: 'Failed to get checks from request.', + }); + } + const { checks }: { checks: string[] } = req.body; + const entityTriplet = stringifyEntityRef({ namespace, kind, name }); + const checkResult = await factChecker.runChecks(entityTriplet, checks); + return res.send(checkResult); + } catch (e) { + return res.status(500).json({ message: e.message }).send(); + } + }); + } else { + logger.info( + 'Starting tech insights module without fact checking endpoints.', + ); + } + + router.get('/fact-schemas', async (req, res) => { + const ids = req.query.ids as string[]; + return res.send(await techInsightsStore.getLatestSchemas(ids)); + }); + + /** + * /facts/latest?entity=component:default/mycomponent&ids[]=factRetrieverId1&ids[]=factRetrieverId2 + */ + router.get('/facts/latest', async (req, res) => { + const { entity } = req.query; + const { namespace, kind, name } = parseEntityName(entity as EntityRef); + const ids = req.query.ids as string[]; + return res.send( + await techInsightsStore.getLatestFactsByIds( + ids, + stringifyEntityRef({ namespace, kind, name }), + ), + ); + }); + + /** + * /facts/latest?entity=component:default/mycomponent&startDateTime=2021-12-24T01:23:45&endDateTime=2021-12-31T23:59:59&ids[]=factRetrieverId1&ids[]=factRetrieverId2 + */ + router.get('/facts/range', async (req, res) => { + const { entity } = req.query; + const { namespace, kind, name } = parseEntityName(entity as EntityRef); + + const ids = req.query.ids as string[]; + const startDatetime = DateTime.fromISO(req.query.startDatetime as string); + const endDatetime = DateTime.fromISO(req.query.endDatetime as string); + if (!startDatetime.isValid || !endDatetime.isValid) { + return res.status(422).send({ + message: 'Failed to parse datetime from request', + field: !startDatetime.isValid ? 'startDateTime' : 'endDateTime', + value: !startDatetime.isValid ? startDatetime : endDatetime, + }); + } + const entityTriplet = stringifyEntityRef({ namespace, kind, name }); + return res.send( + await techInsightsStore.getFactsBetweenTimestampsByIds( + ids, + entityTriplet, + startDatetime, + endDatetime, + ), + ); + }); + return router; +} diff --git a/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts b/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts new file mode 100644 index 0000000000..75e24b7c06 --- /dev/null +++ b/plugins/tech-insights-backend/src/service/techInsightsContextBuilder.ts @@ -0,0 +1,138 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { FactRetrieverEngine } from './fact/FactRetrieverEngine'; +import { Logger } from 'winston'; +import { FactRetrieverRegistry } from './fact/FactRetrieverRegistry'; +import { Config } from '@backstage/config'; +import { + PluginDatabaseManager, + PluginEndpointDiscovery, +} from '@backstage/backend-common'; +import { + FactChecker, + FactCheckerFactory, + FactRetrieverRegistration, + TechInsightCheck, +} from '@backstage/plugin-tech-insights-node'; +import { + initializePersistenceContext, + PersistenceContext, +} from './persistence/persistenceContext'; +import { CheckResult } from '@backstage/plugin-tech-insights-common'; + +/** + * @public + * @typeParam CheckType - Type of the check for the fact checker this builder returns + * @typeParam CheckResultType - Type of the check result for the fact checker this builder returns + * + * Configuration options to initialize TechInsightsBuilder. Generic types params are needed if FactCheckerFactory + * is included for FactChecker creation. + */ +export interface TechInsightsOptions< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + /** + * A collection of FactRetrieverRegistrations. + * Used to register FactRetrievers and their schemas and schedule an execution loop for them. + */ + factRetrievers: FactRetrieverRegistration[]; + + /** + * Optional factory exposing a `construct` method to initialize a FactChecker implementation + */ + factCheckerFactory?: FactCheckerFactory; + + logger: Logger; + config: Config; + discovery: PluginEndpointDiscovery; + database: PluginDatabaseManager; +} + +/** + * @public + * @typeParam CheckType - Type of the check for the fact checker this builder returns + * @typeParam CheckResultType - Type of the check result for the fact checker this builder returns + * + * A container for exported implementations related to TechInsights. + * FactChecker is present if an optional FactCheckerFactory is included in the build stage. + */ +export type TechInsightsContext< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> = { + factChecker?: FactChecker; + persistenceContext: PersistenceContext; +}; + +/** + * @public + * + * Constructs needed persistence context, fact retriever engine + * and optionally fact checker implementations to be used in the tech insights module. + * + * @param options - Needed options to construct TechInsightsContext + * @returns TechInsightsContext with persistence implementations and optionally an implementation of a FactChecker + */ +export const buildTechInsightsContext = async < + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +>( + options: TechInsightsOptions, +): Promise> => { + const { + factRetrievers, + factCheckerFactory, + config, + discovery, + database, + logger, + } = options; + + const factRetrieverRegistry = new FactRetrieverRegistry(factRetrievers); + + const persistenceContext = await initializePersistenceContext( + await database.getClient(), + { logger }, + ); + + const factRetrieverEngine = await FactRetrieverEngine.create({ + repository: persistenceContext.techInsightsStore, + factRetrieverRegistry, + factRetrieverContext: { + config, + discovery, + logger, + }, + }); + + factRetrieverEngine.schedule(); + + if (factCheckerFactory) { + const factChecker = factCheckerFactory.construct( + persistenceContext.techInsightsStore, + ); + return { + persistenceContext, + factChecker, + }; + } + + return { + persistenceContext, + }; +}; diff --git a/plugins/tech-insights-backend/src/setupTests.ts b/plugins/tech-insights-backend/src/setupTests.ts new file mode 100644 index 0000000000..a330613afb --- /dev/null +++ b/plugins/tech-insights-backend/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/plugins/tech-insights-common/.eslintrc.js b/plugins/tech-insights-common/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/tech-insights-common/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/tech-insights-common/README.md b/plugins/tech-insights-common/README.md new file mode 100644 index 0000000000..af14ef0e0d --- /dev/null +++ b/plugins/tech-insights-common/README.md @@ -0,0 +1,3 @@ +# Tech Insights Common + +Common types and functionalities for tech insights shared in an isomorphic manner between BE and FE implementations. diff --git a/plugins/tech-insights-common/api-report.md b/plugins/tech-insights-common/api-report.md new file mode 100644 index 0000000000..ddb38cfa85 --- /dev/null +++ b/plugins/tech-insights-common/api-report.md @@ -0,0 +1,43 @@ +## API Report File for "@backstage/plugin-tech-insights-common" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { DateTime } from 'luxon'; + +// @public +export interface BooleanCheckResult extends CheckResult { + // (undocumented) + result: boolean; +} + +// @public +export interface CheckResponse { + description: string; + factIds: string[]; + id: string; + metadata?: Record; + name: string; + type: string; +} + +// @public +export type CheckResult = { + facts: FactResponse; + check: CheckResponse; +}; + +// @public +export type FactResponse = { + [id: string]: { + id: string; + type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; + description: string; + value: number | string | boolean | DateTime | []; + since?: string; + metadata?: Record; + }; +}; + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/tech-insights-common/package.json b/plugins/tech-insights-common/package.json new file mode 100644 index 0000000000..5e02c9e4b0 --- /dev/null +++ b/plugins/tech-insights-common/package.json @@ -0,0 +1,42 @@ +{ + "name": "@backstage/plugin-tech-insights-common", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/tech-insights-common" + }, + "keywords": [ + "backstage", + "tech-insights" + ], + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@types/luxon": "^2.0.5", + "luxon": "^2.0.2" + }, + "devDependencies": { + "@backstage/cli": "^0.8.0" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/tech-insights-common/src/index.test.ts b/plugins/tech-insights-common/src/index.test.ts new file mode 100644 index 0000000000..0c54118f9a --- /dev/null +++ b/plugins/tech-insights-common/src/index.test.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as anything from './'; + +describe('tech-insights-common', () => { + // Types only + it('should exist', () => { + expect(anything).toBeTruthy(); + }); +}); diff --git a/plugins/tech-insights-common/src/index.ts b/plugins/tech-insights-common/src/index.ts new file mode 100644 index 0000000000..4bfc660748 --- /dev/null +++ b/plugins/tech-insights-common/src/index.ts @@ -0,0 +1,124 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DateTime } from 'luxon'; + +/** + * @public + * + * Response type for checks. + */ +export interface CheckResponse { + /** + * Identifier of the Check + */ + id: string; + /** + * Type identifier for the check. + * Can be used to determine storage options, logical routing to correct FactChecker implementation + * or to help frontend render correct component types based on this + */ + type: string; + /** + * Human readable name of the Check + */ + name: string; + /** + * Description of the Check + */ + description: string; + + /** + * A collection of references to fact rows used to run this checks against + */ + factIds: string[]; + + /** + * Metadata related to a check. + * Can contain links, additional description texts and other actionable data. + * + * Currently loosely typed, but in the future when patterns emerge, key shapes can be defined + */ + metadata?: Record; +} + +/** + * @public + * + * Individual fact response type. + * Keyed by the name of the fact + */ +export type FactResponse = { + [id: string]: { + /** + * Reference and unique identifier of the fact row + */ + id: string; + /** + * Type of the individual fact value + * + * Numbers are split into integers and floating point values. + * `set` indicates a collection of values + */ + type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; + + /** + * Description of the individual fact + */ + description: string; + + /** + * Actual value of the fact + */ + value: number | string | boolean | DateTime | []; + + /** + * An optional SemVer version identifying when this fact was added to the FactSchema + */ + since?: string; + + /** + * Metadata related to an individual fact. + * Can contain links, additional description texts and other actionable data. + * + * Currently loosely typed, but in the future when patterns emerge, key shapes can be defined + */ + metadata?: Record; + }; +}; + +/** + * Generic CheckResult + * + * Contains information about the facts used to calculate the check result + * and information about the check itself. Both may include metadata to be able to display additional information. + * A collection of these should be parseable by the frontend to display scorecards + * + * @public + */ +export type CheckResult = { + facts: FactResponse; + check: CheckResponse; +}; + +/** + * CheckResult of type Boolean. + * + * @public + */ +export interface BooleanCheckResult extends CheckResult { + result: boolean; +} diff --git a/plugins/tech-insights-node/.eslintrc.js b/plugins/tech-insights-node/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/tech-insights-node/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/tech-insights-node/README.md b/plugins/tech-insights-node/README.md new file mode 100644 index 0000000000..4065611837 --- /dev/null +++ b/plugins/tech-insights-node/README.md @@ -0,0 +1,3 @@ +# Tech Insights Node + +Common types and functionalities for tech insights backend implementations to be shared between tech-insights-backend and individual implementations of FactRetrievers, FactCheckers and their persistence options. diff --git a/plugins/tech-insights-node/api-report.md b/plugins/tech-insights-node/api-report.md new file mode 100644 index 0000000000..caf4b4cf0e --- /dev/null +++ b/plugins/tech-insights-node/api-report.md @@ -0,0 +1,149 @@ +## API Report File for "@backstage/plugin-tech-insights-node" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { CheckResult } from '@backstage/plugin-tech-insights-common'; +import { Config } from '@backstage/config'; +import { DateTime } from 'luxon'; +import { Logger as Logger_2 } from 'winston'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; + +// @public +export type CheckValidationResponse = { + valid: boolean; + message?: string; + errors?: unknown[]; +}; + +// @public +export interface FactChecker< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + getChecks(): Promise; + runChecks(entity: string, checks?: string[]): Promise; + validate(check: CheckType): Promise; +} + +// @public +export interface FactCheckerFactory< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + // (undocumented) + construct( + repository: TechInsightsStore, + ): FactChecker; +} + +// @public +export interface FactRetriever { + entityFilter?: + | Record[] + | Record; + handler: (ctx: FactRetrieverContext) => Promise; + id: string; + schema: FactSchema; + version: string; +} + +// @public +export type FactRetrieverContext = { + config: Config; + discovery: PluginEndpointDiscovery; + logger: Logger_2; +}; + +// @public +export type FactRetrieverRegistration = { + factRetriever: FactRetriever; + cadence?: string; +}; + +// @public +export type FactSchema = { + [name: string]: { + type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; + description: string; + since?: string; + metadata?: Record; + }; +}; + +// @public +export type FactSchemaDefinition = Omit; + +// @public +export type FlatTechInsightFact = TechInsightFact & { + id: string; +}; + +// @public +export interface TechInsightCheck { + description: string; + factIds: string[]; + failureMetadata?: Record; + id: string; + name: string; + successMetadata?: Record; + type: string; +} + +// @public +export interface TechInsightCheckRegistry { + // (undocumented) + get(checkId: string): Promise; + // (undocumented) + getAll(checks: string[]): Promise; + // (undocumented) + list(): Promise; + // (undocumented) + register(check: CheckType): Promise; +} + +// @public +export type TechInsightFact = { + entity: { + namespace: string; + kind: string; + name: string; + }; + facts: Record< + string, + | number + | string + | boolean + | DateTime + | number[] + | string[] + | boolean[] + | DateTime[] + >; + timestamp?: DateTime; +}; + +// @public +export interface TechInsightsStore { + getFactsBetweenTimestampsByIds( + ids: string[], + entity: string, + startDateTime: DateTime, + endDateTime: DateTime, + ): Promise<{ + [factRef: string]: FlatTechInsightFact[]; + }>; + // (undocumented) + getLatestFactsByIds( + ids: string[], + entity: string, + ): Promise<{ + [factRef: string]: FlatTechInsightFact; + }>; + getLatestSchemas(ids?: string[]): Promise; + insertFacts(id: string, facts: TechInsightFact[]): Promise; + insertFactSchema(schemaDefinition: FactSchemaDefinition): Promise; +} + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/tech-insights-node/package.json b/plugins/tech-insights-node/package.json new file mode 100644 index 0000000000..4a7cf3589b --- /dev/null +++ b/plugins/tech-insights-node/package.json @@ -0,0 +1,46 @@ +{ + "name": "@backstage/plugin-tech-insights-node", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/tech-insights-node" + }, + "keywords": [ + "backstage", + "tech-insights" + ], + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/backend-common": "^0.9.0", + "@backstage/config": "^0.1.8", + "@backstage/plugin-tech-insights-common": "^0.1.0 ", + "@types/luxon": "^2.0.5", + "luxon": "^2.0.2", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/cli": "^0.8.0" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/tech-insights-node/src/checks.ts b/plugins/tech-insights-node/src/checks.ts new file mode 100644 index 0000000000..e5f10aff6d --- /dev/null +++ b/plugins/tech-insights-node/src/checks.ts @@ -0,0 +1,153 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { TechInsightsStore } from './persistence'; +import { CheckResult } from '@backstage/plugin-tech-insights-common'; + +/** + * A factory wrapper to construct FactChecker implementations. + * + * @public + * @typeParam CheckType - Implementation specific Check. Can extend TechInsightCheck with additional information + * @typeParam CheckResultType - Implementation specific result of a check. Can extend CheckResult with additional information + */ +export interface FactCheckerFactory< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + /** + * @param repository - TechInsightsStore + * @returns an implementation of a FactChecker for generic types defined in the factory + */ + construct( + repository: TechInsightsStore, + ): FactChecker; +} + +/** + * FactChecker interface + * + * A generic interface that can be implemented to create checkers for specific check and check return types. + * This is used especially when creating Scorecards and displaying results of rules when run against facts. + * + * @public + * @typeParam CheckType - Implementation specific Check. Can extend TechInsightCheck with additional information + * @typeParam CheckResultType - Implementation specific result of a check. Can extend CheckResult with additional information + */ +export interface FactChecker< + CheckType extends TechInsightCheck, + CheckResultType extends CheckResult, +> { + /** + * Runs checks against an entity. + * + * @param entity - A reference to an entity to run checks against. In a format namespace/kind/name + * @param checks - A collection of checks to run against provided entity + * @returns - A collection containing check/fact information and the actual results of the check + */ + runChecks(entity: string, checks?: string[]): Promise; + + /** + * Retrieves all available checks that can be used to run checks against. + * The implementation can be just a piping through to CheckRegistry implementation if such is in use. + * + * @returns - A collection of checks + */ + getChecks(): Promise; + + /** + * Validates if check is valid and can be run with the current implementation + * + * @param check - The check to be validated + * @returns - Validation result + */ + validate(check: CheckType): Promise; +} + +/** + * Registry containing checks for tech insights. + * + * @public + * @typeParam CheckType - Implementation specific Check. Can extend TechInsightCheck with additional information + * + */ +export interface TechInsightCheckRegistry { + register(check: CheckType): Promise; + get(checkId: string): Promise; + getAll(checks: string[]): Promise; + list(): Promise; +} + +/** + * Generic definition of a check for Tech Insights + * + * @public + */ +export interface TechInsightCheck { + /** + * Unique identifier of the check + * + * Used to identify which checks to use when running checks. + */ + id: string; + + /** + * Type identifier for the check. + * Can be used to determine storage options, logical routing to correct FactChecker implementation + * or to help frontend render correct component types based on this + */ + type: string; + + /** + * Human readable name of the check, may be displayed in the UI + */ + name: string; + + /** + * Human readable description of the check, may be displayed in the UI + */ + description: string; + + /** + * A collection of string referencing fact rows that a check will be run against. + * + * References the fact container, aka fact retriever itself which may or may not contain multiple individual facts and values + */ + factIds: string[]; + + /** + * Metadata to be returned in case a check has been successfully evaluated + * Can contain links, description texts or other actionable items + */ + successMetadata?: Record; + + /** + * Metadata to be returned in case a check evaluation has ended in failure + * Can contain links, description texts or other actionable items + */ + failureMetadata?: Record; +} + +/** + * Validation response from CheckValidator + * + * May contain additional data for display purposes + * @public + */ +export type CheckValidationResponse = { + valid: boolean; + message?: string; + errors?: unknown[]; +}; diff --git a/plugins/tech-insights-node/src/facts.ts b/plugins/tech-insights-node/src/facts.ts new file mode 100644 index 0000000000..a52a91f2f4 --- /dev/null +++ b/plugins/tech-insights-node/src/facts.ts @@ -0,0 +1,216 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { DateTime } from 'luxon'; +import { Config } from '@backstage/config'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Logger } from 'winston'; + +/** + * A container for facts. The shape of the fact records needs to correspond to the FactSchema with same `ref` value. + * Each container contains a reference to an entity which can be a Backstage entity or a generic construct + * outside of backstage with same shape. + * + * Container may contain multiple individual facts and their values + * + * @public + */ +export type TechInsightFact = { + /** + * Entity reference that this fact relates to + */ + entity: { + namespace: string; + kind: string; + name: string; + }; + + /** + * A collection of fact values as key value pairs. + * + * Key indicates fact name as it is defined in FactSchema + */ + facts: Record< + string, + | number + | string + | boolean + | DateTime + | number[] + | string[] + | boolean[] + | DateTime[] + >; + + /** + * Optional timestamp value which can be used to override retrieval time of the fact row. + * Otherwise when stored into data storage, defaults to current time + */ + timestamp?: DateTime; +}; + +/** + + * Response type used when returning from database and API. + * Adds a field for ref for easier usage + * + * @public + */ +export type FlatTechInsightFact = TechInsightFact & { + /** + * Reference and unique identifier of the fact row + */ + id: string; +}; + +/** + * @public + * + * A record type to specify individual fact shapes + * + * Used as part of a schema to validate, identify and generically construct usage implementations + * of individual fact values in the system. + */ +export type FactSchema = { + /** + * Name of the fact + */ + [name: string]: { + /** + * Type of the individual fact value + * + * Numbers are split into integers and floating point values. + * `set` indicates a collection of values + */ + type: 'integer' | 'float' | 'string' | 'boolean' | 'datetime' | 'set'; + + /** + * A description of this individual fact value + */ + description: string; + + /** + * Optional semver string to indicate when this specific fact definition was added to the schema + */ + since?: string; + + /** + * Metadata related to an individual fact. + * Can contain links, additional description texts and other actionable data. + * + * Currently loosely typed, but in the future when patterns emerge, key shapes can be defined + * + * examples: + * ``` + * \{ + * link: 'https://sonarqube.mycompany.com/fix-these-issues', + * suggestion: 'To affect this value, you can do x, y, z', + * minValue: 0 + * \} + * ``` + */ + metadata?: Record; + }; +}; + +/** + * @public + * + * FactRetrieverContext injected into individual handler methods of FactRetriever implementations. + * The context can be used to construct logic to retrieve entities, contact integration points + * and fetch and calculate fact values from external sources. + */ +export type FactRetrieverContext = { + config: Config; + discovery: PluginEndpointDiscovery; + logger: Logger; +}; + +/** + * @public + * + * FactRetriever interface + * + * A component specifying + */ +export interface FactRetriever { + /** + * A unique identifier of the retriever. + * Used to identify and store individual facts returned from this retriever + * and schemas defined by this retriever. + */ + id: string; + + /** + * Semver string indicating the version of this fact retriever + * This version is used to determine if the schema this fact retriever matches the data this fact retriever provides. + * + * Should be incremented on changes to returned data from the handler or if the schema changes. + */ + version: string; + + /** + * Handler function that needs to be implemented to retrieve fact values for entities. + * + * @param ctx - FactRetrieverContext which can be used to retrieve config and contact integrations + * @returns - A collection of TechInsightFacts grouped by entities. + */ + handler: (ctx: FactRetrieverContext) => Promise; + + /** + * A fact schema defining the shape of data returned from the handler method for each entity + */ + schema: FactSchema; + + /** + * An optional object/array of objects of entity filters to indicate if this fact retriever is valid for an entity type. + * If omitted, the retriever should apply to all entities. + * + * Should be defined for example: + * \{ field: 'kind', values: \['component'\] \} + * \{ field: 'metadata.name', values: \['component-1', 'component-2'\] \} + */ + entityFilter?: + | Record[] + | Record; +} + +/** + * @public + * + * A flat serializable structure for Facts. + * Containing information about fact schema, version, id, and entity filters + */ +export type FactSchemaDefinition = Omit; + +/** + * @public + * + * Registration of a fact retriever + * Used to add and schedule individual fact retrievers to the fact retriever engine. + */ +export type FactRetrieverRegistration = { + /** + * Actual FactRetriever implementation + */ + factRetriever: FactRetriever; + + /** + * Cron expression to indicate when the retriever should be triggered. + * Defaults to a random point in time every 24h + * + */ + cadence?: string; +}; diff --git a/plugins/tech-insights-node/src/index.test.ts b/plugins/tech-insights-node/src/index.test.ts new file mode 100644 index 0000000000..5e5f8c9d77 --- /dev/null +++ b/plugins/tech-insights-node/src/index.test.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as anything from './'; + +describe('tech-insights-node', () => { + // Types only + it('should exist', () => { + expect(anything).toBeTruthy(); + }); +}); diff --git a/plugins/tech-insights-node/src/index.ts b/plugins/tech-insights-node/src/index.ts new file mode 100644 index 0000000000..70ef27e159 --- /dev/null +++ b/plugins/tech-insights-node/src/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './checks'; +export * from './facts'; +export * from './persistence'; diff --git a/plugins/tech-insights-node/src/persistence.ts b/plugins/tech-insights-node/src/persistence.ts new file mode 100644 index 0000000000..444e961008 --- /dev/null +++ b/plugins/tech-insights-node/src/persistence.ts @@ -0,0 +1,83 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + FactSchema, + TechInsightFact, + FlatTechInsightFact, + FactSchemaDefinition, +} from './facts'; +import { DateTime } from 'luxon'; + +/** + * TechInsights Database + * + * @public + */ +export interface TechInsightsStore { + /** + * Stores fact containers as rows into data store. + * Individual items in array correspond to a fact schema based on reference and entity based on entity identifier. + * + * Each row may contain multiple individual facts and values + * + * @param id - Unique identifier of the fact retriever these facts relate to + * @param facts - A collection of TechInsightFacts + */ + insertFacts(id: string, facts: TechInsightFact[]): Promise; + + /** + * @param ids - A collection of fact row identifiers + * @param entity - A string identifying an entity. In a format namespace/kind/name + * + * @returns - An object keyed by a fact reference and containing an individual TechInsightFact + */ + getLatestFactsByIds( + ids: string[], + entity: string, + ): Promise<{ [factRef: string]: FlatTechInsightFact }>; + + /** + * Retrieves fact values identified by fact row references for an individual entity. + * + * @param ids - A collection of fact row identifiers + * @param entity - A string identifying an entity. In a format namespace/kind/name + * @param startDateTime - DateTime object indicating start of the time frame + * @param endDateTime - DateTime object indicating start of the time frame + * + * @returns - An object keyed by a fact reference and containing a collection of TechInsightFacts matching the time frame + */ + getFactsBetweenTimestampsByIds( + ids: string[], + entity: string, + startDateTime: DateTime, + endDateTime: DateTime, + ): Promise<{ [factRef: string]: FlatTechInsightFact[] }>; + + /** + * Stores versioned fact schemas into data store + * + * @param schemaDefinition - FactSchemaDefinition containing id, version, schema and entityTypes. + */ + insertFactSchema(schemaDefinition: FactSchemaDefinition): Promise; + + /** + * Retrieves latest versions (as defined by semver) of fact schemas from the data store. + * + * @param ids - Collection of ids to return. If omitted, all Schemas should be returned. + * @returns - A collection of schemas + */ + getLatestSchemas(ids?: string[]): Promise; +} diff --git a/plugins/tech-insights-node/src/setupTests.ts b/plugins/tech-insights-node/src/setupTests.ts new file mode 100644 index 0000000000..a330613afb --- /dev/null +++ b/plugins/tech-insights-node/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/plugins/techdocs-backend/CHANGELOG.md b/plugins/techdocs-backend/CHANGELOG.md index 103306f79b..7331ea0147 100644 --- a/plugins/techdocs-backend/CHANGELOG.md +++ b/plugins/techdocs-backend/CHANGELOG.md @@ -1,5 +1,11 @@ # @backstage/plugin-techdocs-backend +## 0.10.7 + +### Patch Changes + +- b45607a2ec: Make techdocs s3 publisher credentials config schema optional. + ## 0.10.6 ### Patch Changes diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts index ac31474fa9..0023a4b757 100644 --- a/plugins/techdocs-backend/config.d.ts +++ b/plugins/techdocs-backend/config.d.ts @@ -83,12 +83,12 @@ export interface Config { * User access key id * @visibility secret */ - accessKeyId: string; + accessKeyId?: string; /** * User secret access key * @visibility secret */ - secretAccessKey: string; + secretAccessKey?: string; /** * ARN of role to be assumed * @visibility backend diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 3208b451fa..c71833948b 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/plugin-techdocs-backend", "description": "The Backstage backend plugin that renders technical documentation for your components", - "version": "0.10.6", + "version": "0.10.7", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", diff --git a/plugins/user-settings/package.json b/plugins/user-settings/package.json index 9a3b65f7ae..9ec642892b 100644 --- a/plugins/user-settings/package.json +++ b/plugins/user-settings/package.json @@ -45,7 +45,6 @@ "devDependencies": { "@backstage/cli": "^0.8.1", "@backstage/core-app-api": "^0.1.19", - "@backstage/core-plugin-api": "^0.1.11", "@backstage/dev-utils": "^0.2.12", "@backstage/test-utils": "^0.1.20", "@testing-library/jest-dom": "^5.10.1", diff --git a/yarn.lock b/yarn.lock index 47e9b756b4..e88a6ca99e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -266,13 +266,20 @@ dependencies: tslib "^2.0.0" -"@azure/msal-common@^4.0.0", "@azure/msal-common@^4.4.0": +"@azure/msal-common@^4.0.0": version "4.4.0" resolved "https://registry.npmjs.org/@azure/msal-common/-/msal-common-4.4.0.tgz#818526042f78838ebc332fb735e7de64d8bccb45" integrity sha512-Qrs33Ctt2KM7NxArFPIUKc8UbIcm7zYxJFdJeQ9k7HKBhVk3e88CUz1Mw33cS/Jr+YA1H02OAzHg++bJ+4SFyQ== dependencies: debug "^4.1.1" +"@azure/msal-common@^5.0.1": + version "5.0.1" + resolved "https://registry.npmjs.org/@azure/msal-common/-/msal-common-5.0.1.tgz#234030b340cc63575e84190b96a8a336b69c7698" + integrity sha512-CmPR3XM9+CGUu7V/+bAwDxyN6XqWJJhVLmv7utT3sbgay4l5roVXsD1t4wURTs8PwzxmmnJOrhvvGhoDxUW69g== + dependencies: + debug "^4.1.1" + "@azure/msal-node@1.0.0-beta.6": version "1.0.0-beta.6" resolved "https://registry.npmjs.org/@azure/msal-node/-/msal-node-1.0.0-beta.6.tgz#da6bc3a3a861057c85586055960e069f162548ee" @@ -284,12 +291,12 @@ uuid "^8.3.0" "@azure/msal-node@^1.1.0": - version "1.2.0" - resolved "https://registry.npmjs.org/@azure/msal-node/-/msal-node-1.2.0.tgz#d08a5fb3391436715cb37c6eb44816013bdfa11e" - integrity sha512-79o5n483vslc7Qegh9+0BsxODRmlk6YYjVdl9jvwmAuF+i+oylq57e7RVhTVocKCbLCIMOKARI14JyKdDbW0WA== + version "1.3.2" + resolved "https://registry.npmjs.org/@azure/msal-node/-/msal-node-1.3.2.tgz#17665397c04b9cad57b42eb6e21d5d6f35d9b83a" + integrity sha512-aKU2lVRKhZa1IJ/Za/Ir6qlythQ3FHz0g0px3SbM4iC1otyr3ANS4mIn/6fmkpZDIHc8eAgJh2KMep1Yn2zpig== dependencies: - "@azure/msal-common" "^4.4.0" - axios "^0.21.1" + "@azure/msal-common" "^5.0.1" + axios "^0.21.4" jsonwebtoken "^8.5.1" uuid "^8.3.0" @@ -2165,7 +2172,7 @@ "@babel/helper-validator-identifier" "^7.14.9" to-fast-properties "^2.0.0" -"@backstage/catalog-client@^0.3.18": +"@backstage/catalog-client@^0.3.16", "@backstage/catalog-client@^0.3.18": version "0.3.19" resolved "https://registry.npmjs.org/@backstage/catalog-client/-/catalog-client-0.3.19.tgz#109b0fde1cc2d9a306635f3775fc8f6174172b14" integrity sha512-0ydcC/1ISNgR8SggOUnMNNHtsRwrKN5GX+AXgTVdZk4qpz1MqG1+fzrNld9V7KVvXdsbGDZAl/b5a7/FJEpCqg== @@ -5184,15 +5191,16 @@ react-use "^17.2.4" "@roadiehq/backstage-plugin-github-insights@^1.1.23": - version "1.1.23" - resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-insights/-/backstage-plugin-github-insights-1.1.23.tgz#bcf7424df5a659df39d5aa75f5fffd8e02c9437e" - integrity sha512-3HL9nEtlaE+gFmV0xkrtZGbWHbS26z+NHzEk2KIyV8oqReQ+57PRLTVcXCjFA+djkKfP2hpu5s7zp72aVb0v+w== + version "1.2.2" + resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-insights/-/backstage-plugin-github-insights-1.2.2.tgz#09d958ed15adbb598afda34187f45deedec2264d" + integrity sha512-ybomCU/3NIQJYahp/E78wl+JuzpwkRkhyeCcySLA45GaCz/mEqAGSs4xDZancpV2ls6nxV3mKUw/DL9afmMGAw== dependencies: "@backstage/catalog-model" "^0.9.0" "@backstage/core-app-api" "^0.1.3" - "@backstage/core-components" "^0.3.0" + "@backstage/core-components" "^0.7.0" "@backstage/core-plugin-api" "^0.1.3" - "@backstage/plugin-catalog-react" "^0.4.0" + "@backstage/integration-react" "^0.1.10" + "@backstage/plugin-catalog-react" "^0.6.0" "@backstage/theme" "^0.2.7" "@date-io/core" "2.10.7" "@material-ui/core" "^4.11.0" @@ -7220,6 +7228,11 @@ resolved "https://registry.npmjs.org/@types/luxon/-/luxon-2.0.4.tgz#f7b5a86ccd843c0ccaddfaedd9ee1081bc1cde3b" integrity sha512-l3xuhmyF2kBldy15SeY6d6HbK2BacEcSK1qTF1ISPtPHr29JH0C1fndz9ExXLKpGl0J6pZi+dGp1i5xesMt60Q== +"@types/luxon@^2.0.5": + version "2.0.5" + resolved "https://registry.npmjs.org/@types/luxon/-/luxon-2.0.5.tgz#29d3b095d55ee50df8f4cf109b16009334d9828e" + integrity sha512-GKrG5v16BOs9XGpouu33hOkAFaiSDi3ZaDXG9F2yAoyzHRBtksZnI60VWY5aM/yAENCccBejrxw8jDY+9OVlxw== + "@types/markdown-to-jsx@^6.11.3": version "6.11.3" resolved "https://registry.npmjs.org/@types/markdown-to-jsx/-/markdown-to-jsx-6.11.3.tgz#cdd1619308fecbc8be7e6a26f3751260249b020e" @@ -7292,6 +7305,18 @@ resolved "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197" integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== +"@types/node-cron@^2.0.4": + version "2.0.5" + resolved "https://registry.npmjs.org/@types/node-cron/-/node-cron-2.0.5.tgz#e244709a86d32453c5a702ced35b53db683fbc8e" + integrity sha512-rQ4kduTmgW11tbtx0/RsoybYHHPu4Vxw5v5ZS5qUKNerlEAI8r8P1F5UUZ2o2HTvzG759sbFxuRuqWxU8zc+EQ== + dependencies: + "@types/tz-offset" "*" + +"@types/node-cron@^3.0.0": + version "3.0.0" + resolved "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.0.tgz#f946cefb5c05c64f460090f6be97bd50460c8898" + integrity sha512-RNBIyVwa/1v2r8/SqK8tadH2sJlFRAo5Ghac/cOcCv4Kp94m0I03UmAh9WVhCqS9ZdB84dF3x47p9aTw8E4c4A== + "@types/node-fetch@^2.5.0", "@types/node-fetch@^2.5.7": version "2.5.8" resolved "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.8.tgz#e199c835d234c7eb0846f6618012e558544ee2fb" @@ -7439,6 +7464,13 @@ resolved "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.4.tgz#fcf7205c25dff795ee79af1e30da2c9790808f11" integrity sha512-rZ5drC/jWjrArrS8BR6SIr4cWpW09RNTYt9AMZo3Jwwif+iacXAqgVjm0B0Bv/S1jhDXKHqRVNCbACkJ89RAnQ== +"@types/puppeteer@^5.4.4": + version "5.4.4" + resolved "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-5.4.4.tgz#e92abeccc4f46207c3e1b38934a1246be080ccd0" + integrity sha512-3Nau+qi69CN55VwZb0ATtdUAlYlqOOQ3OfQfq0Hqgc4JMFXiQT/XInlwQ9g6LbicDslE6loIFsXFklGh5XmI6Q== + dependencies: + "@types/node" "*" + "@types/q@^1.5.1": version "1.5.2" resolved "https://registry.npmjs.org/@types/q/-/q-1.5.2.tgz#690a1475b84f2a884fd07cd797c00f5f31356ea8" @@ -7627,6 +7659,11 @@ resolved "https://registry.npmjs.org/@types/semver/-/semver-6.2.1.tgz#a236185670a7860f1597cf73bea2e16d001461ba" integrity sha512-+beqKQOh9PYxuHvijhVl+tIHvT6tuwOrE9m14zd+MT2A38KoKZhh7pYJ0SNleLtwDsiIxHDsIk9bv01oOxvSvA== +"@types/semver@^7.3.8": + version "7.3.8" + resolved "https://registry.npmjs.org/@types/semver/-/semver-7.3.8.tgz#508a27995498d7586dcecd77c25e289bfaf90c59" + integrity sha512-D/2EJvAlCEtYFEYmmlGwbGXuK886HzyCc3nZX/tkFTQdEU8jZDAgiv08P162yB17y4ZXZoq7yFAnW4GDBb9Now== + "@types/serve-static@*": version "1.13.9" resolved "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.9.tgz#aacf28a85a05ee29a11fb7c3ead935ac56f33e4e" @@ -7798,6 +7835,11 @@ dependencies: "@types/node" "*" +"@types/tz-offset@*": + version "0.0.0" + resolved "https://registry.npmjs.org/@types/tz-offset/-/tz-offset-0.0.0.tgz#d58f1cebd794148d245420f8f0660305d320e565" + integrity sha512-XLD/llTSB6EBe3thkN+/I0L+yCTB6sjrcVovQdx2Cnl6N6bTzHmwe/J8mWnsXFgxLrj/emzdv8IR4evKYG2qxQ== + "@types/uglify-js@*": version "3.0.4" resolved "https://registry.npmjs.org/@types/uglify-js/-/uglify-js-3.0.4.tgz#96beae23df6f561862a830b4288a49e86baac082" @@ -7938,6 +7980,13 @@ resolved "https://registry.npmjs.org/@types/yarnpkg__lockfile/-/yarnpkg__lockfile-1.1.4.tgz#445251eb00bd9c1e751f82c7c6bf4f714edfd464" integrity sha512-/emrKCfQMQmFCqRqqBJ0JueHBT06jBRM3e8OgnvDUcvuExONujIk2hFA5dNsN9Nt41ljGVDdChvCydATZ+KOZw== +"@types/yauzl@^2.9.1": + version "2.9.2" + resolved "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.2.tgz#c48e5d56aff1444409e39fa164b0b4d4552a7b7a" + integrity sha512-8uALY5LTvSuHgloDVUvWP3pIauILm+8/0pDMokuDYIoNsOkSwd5AiHBTSEJjKTDcZr5z8UpgOWZkxBF4iJftoA== + dependencies: + "@types/node" "*" + "@types/yup@^0.29.13": version "0.29.13" resolved "https://registry.npmjs.org/@types/yup/-/yup-0.29.13.tgz#21b137ba60841307a3c8a1050d3bf4e63ad561e9" @@ -8362,7 +8411,7 @@ a-sync-waterfall@^1.0.0: resolved "https://registry.npmjs.org/a-sync-waterfall/-/a-sync-waterfall-1.0.1.tgz#75b6b6aa72598b497a125e7a2770f14f4c8a1fa7" integrity sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA== -abab@^2.0.0, abab@^2.0.3: +abab@^2.0.3: version "2.0.3" resolved "https://registry.npmjs.org/abab/-/abab-2.0.3.tgz#623e2075e02eb2d3f2475e49f99c91846467907a" integrity sha512-tsFzPpcttalNjFBCFMqsKYQcWxxen1pgJR56by//QwvJc4/OUS3kPOOttx2tSIfjsylB0pYu7f5D3K1RCxUnUg== @@ -8392,14 +8441,6 @@ accepts@^1.3.5, accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.7: mime-types "~2.1.24" negotiator "0.6.2" -acorn-globals@^4.1.0: - version "4.3.4" - resolved "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.3.4.tgz#9fa1926addc11c97308c4e66d7add0d40c3272e7" - integrity sha512-clfQEh21R+D0leSbUdWf3OcfqyaCSAQ8Ryq00bofSekfr9W8u1jyYZo6ir0xu9Gtcf7BjcHJpnbZH7JOCpP60A== - dependencies: - acorn "^6.0.1" - acorn-walk "^6.0.1" - acorn-globals@^6.0.0: version "6.0.0" resolved "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz#46cdd39f0f8ff08a876619b55f5ac8a6dc770b45" @@ -8418,11 +8459,6 @@ acorn-jsx@^5.3.1: resolved "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.1.tgz#fc8661e11b7ac1539c47dbfea2e72b3af34d267b" integrity sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng== -acorn-walk@^6.0.1: - version "6.2.0" - resolved "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.2.0.tgz#123cb8f3b84c2171f1f7fb252615b1c78a6b1a8c" - integrity sha512-7evsyfH1cLOCdAzZAd43Cic04yKydNx0cF+7tiA19p1XnLLPU4dpCQOqpjqwokFe//vS0QqfqqjCS2JkiIs0cA== - acorn-walk@^7.1.1: version "7.1.1" resolved "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.1.1.tgz#345f0dffad5c735e7373d2fec9a1023e6a44b83e" @@ -8433,12 +8469,7 @@ acorn-walk@^8.1.1: resolved "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== -acorn@^5.5.3: - version "5.7.4" - resolved "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz#3e8d8a9947d0599a1796d10225d7432f4a4acf5e" - integrity sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg== - -acorn@^6.0.1, acorn@^6.4.1: +acorn@^6.4.1: version "6.4.1" resolved "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz#531e58ba3f51b9dacb9a6646ca4debf5b14ca474" integrity sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA== @@ -9009,11 +9040,6 @@ array-differ@^3.0.0: resolved "https://registry.npmjs.org/array-differ/-/array-differ-3.0.0.tgz#3cbb3d0f316810eafcc47624734237d6aee4ae6b" integrity sha512-THtfYS6KtME/yIAhKjZ2ul7XI96lQGHRputJQHO80LAWQnuGP4iCIN8vdMRboGbIEYBwU33q8Tch1os2+X0kMg== -array-equal@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz#8c2a5ef2472fd9ea742b04c77a75093ba2757c93" - integrity sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM= - array-filter@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/array-filter/-/array-filter-1.0.0.tgz#baf79e62e6ef4c2a4c0b831232daffec251f9d83" @@ -9176,11 +9202,6 @@ async-each@^1.0.1: resolved "https://registry.npmjs.org/async-each/-/async-each-1.0.3.tgz#b727dbf87d7651602f06f4d4ac387f47d91b0cbf" integrity sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ== -async-limiter@~1.0.0: - version "1.0.1" - resolved "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd" - integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ== - async-lock@^1.1.0: version "1.2.4" resolved "https://registry.npmjs.org/async-lock/-/async-lock-1.2.4.tgz#80d0d612383045dd0c30eb5aad08510c1397cb91" @@ -9348,7 +9369,7 @@ axios-cached-dns-resolve@0.5.2: pino "^5.12.2" pino-pretty "^2.6.0" -axios@^0.21.1: +axios@^0.21.1, axios@^0.21.4: version "0.21.4" resolved "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== @@ -9611,7 +9632,7 @@ babel-preset-jest@^26.6.2: babel-plugin-jest-hoist "^26.6.2" babel-preset-current-node-syntax "^1.0.0" -babel-runtime@6.26.0, babel-runtime@^6.26.0: +babel-runtime@^6.26.0: version "6.26.0" resolved "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz#965c7058668e82b55d7bfe04ff2337bc8b5647fe" integrity sha1-llxwWGaOgrVde/4E/yM3vItWR/4= @@ -9828,7 +9849,7 @@ block-stream@*: dependencies: inherits "~2.0.0" -bluebird@3.7.2, bluebird@^3.3.5, bluebird@^3.5.1, bluebird@^3.5.5, bluebird@^3.7.2: +bluebird@3.7.2, bluebird@^3.3.5, bluebird@^3.5.5, bluebird@^3.7.2: version "3.7.2" resolved "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== @@ -10097,7 +10118,7 @@ buffer@4.9.2, buffer@^4.3.0: ieee754 "^1.1.4" isarray "^1.0.0" -buffer@^5.5.0, buffer@^5.7.0: +buffer@^5.2.1, buffer@^5.5.0, buffer@^5.7.0: version "5.7.1" resolved "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== @@ -10839,7 +10860,7 @@ clone-stats@^1.0.0: resolved "https://registry.npmjs.org/clone-stats/-/clone-stats-1.0.0.tgz#b3782dff8bb5474e18b9b6bf0fdfe782f8777680" integrity sha1-s3gt/4u1R04Yuba/D9/ngvh3doA= -clone@2.x, clone@^2.1.1: +clone@2.x, clone@^2.1.1, clone@^2.1.2: version "2.1.2" resolved "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f" integrity sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18= @@ -11918,22 +11939,15 @@ csso@^4.0.2: dependencies: css-tree "1.0.0-alpha.37" -cssom@0.3.x, "cssom@>= 0.3.2 < 0.4.0", cssom@~0.3.6: - version "0.3.8" - resolved "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz#9f1276f5b2b463f2114d3f2c75250af8c1a36f4a" - integrity sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg== - cssom@^0.4.4: version "0.4.4" resolved "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz#5a66cf93d2d0b661d80bf6a44fb65f5c2e4e0a10" integrity sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw== -cssstyle@^1.0.0: - version "1.4.0" - resolved "https://registry.npmjs.org/cssstyle/-/cssstyle-1.4.0.tgz#9d31328229d3c565c61e586b02041a28fccdccf1" - integrity sha512-GBrLZYZ4X4x6/QEoBnIrqb8B/f5l4+8me2dkom/j1Gtbxy0kBv6OGzKuAsGM75bkGwGAFkt56Iwg28S3XTZgSA== - dependencies: - cssom "0.3.x" +cssom@~0.3.6: + version "0.3.8" + resolved "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz#9f1276f5b2b463f2114d3f2c75250af8c1a36f4a" + integrity sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg== cssstyle@^2.2.0: version "2.3.0" @@ -12297,15 +12311,6 @@ dashify@^2.0.0: resolved "https://registry.npmjs.org/dashify/-/dashify-2.0.0.tgz#fff270ca2868ca427fee571de35691d6e437a648" integrity sha512-hpA5C/YrPjucXypHPPc0oJ1l9Hf6wWbiOL7Ik42cxnsUOhWiCB/fylKbKqqJalW9FgkNQCw16YO8uW9Hs0Iy1A== -data-urls@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz#15ee0582baa5e22bb59c77140da8f9c76963bbfe" - integrity sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ== - dependencies: - abab "^2.0.0" - whatwg-mimetype "^2.2.0" - whatwg-url "^7.0.0" - data-urls@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz#156485a72963a970f5d5821aaf642bef2bf2db9b" @@ -12652,6 +12657,11 @@ detect-port@^1.3.0: address "^1.0.1" debug "^2.6.0" +devtools-protocol@0.0.901419: + version "0.0.901419" + resolved "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.901419.tgz#79b5459c48fe7e1c5563c02bd72f8fec3e0cebcd" + integrity sha512-4INMPwNm9XRpBukhNbF7OB6fNTTCaI8pzy/fXg0xQzAy5h3zL1P8xT3QazgKqBrb/hAYwIBizqDBZ7GtJE74QQ== + dezalgo@^1.0.0: version "1.0.3" resolved "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.3.tgz#7f742de066fc748bc8db820569dddce49bf0d456" @@ -12846,13 +12856,6 @@ domelementtype@^2.0.1, domelementtype@^2.1.0: resolved "https://registry.npmjs.org/domelementtype/-/domelementtype-2.1.0.tgz#a851c080a6d1c3d94344aed151d99f669edf585e" integrity sha512-LsTgx/L5VpD+Q8lmsXSHW2WpA+eBlZ9HPf3erD1IoPF00/3JKHZ3BknUVA2QGDNu69ZNmyFmCWBSO45XjYKC5w== -domexception@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz#937442644ca6a31261ef36e3ec677fe805582c90" - integrity sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug== - dependencies: - webidl-conversions "^4.0.2" - domexception@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz#fb44aefba793e1574b0af6aed2801d057529f304" @@ -13385,7 +13388,7 @@ escape-string-regexp@^5.0.0: resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz#4683126b500b61762f2dbebace1806e8be31b1c8" integrity sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw== -escodegen@^1.14.1, escodegen@^1.9.1: +escodegen@^1.14.1: version "1.14.3" resolved "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz#4e7b81fba61581dc97582ed78cab7f0e8d63f503" integrity sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw== @@ -13723,6 +13726,11 @@ eventemitter2@^6.4.3: resolved "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.4.tgz#aa96e8275c4dbeb017a5d0e03780c65612a1202b" integrity sha512-HLU3NDY6wARrLCEwyGKRBvuWYyvW6mHYv72SJJAH3iJN3a6eVUvkjFkcxah1bcTgGVBBrFdIopBJPhCQFMLyXw== +eventemitter2@^6.4.4: + version "6.4.5" + resolved "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.5.tgz#97380f758ae24ac15df8353e0cc27f8b95644655" + integrity sha512-bXE7Dyc1i6oQElDG0jMRZJrRAn9QR2xyyFGmBdZleNmyQX0FqGYmhZIrIrpPfm/w//LTo4tVQGOGQcGCb5q9uw== + eventemitter3@^3.1.0: version "3.1.2" resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7" @@ -13748,13 +13756,6 @@ events@^3.0.0, events@^3.2.0, events@^3.3.0: resolved "https://registry.npmjs.org/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== -eventsource@^1.0.5: - version "1.0.7" - resolved "https://registry.npmjs.org/eventsource/-/eventsource-1.0.7.tgz#8fbc72c93fcd34088090bc0a4e64f4b5cee6d8d0" - integrity sha512-4Ln17+vVT0k8aWq+t/bF5arcS3EpT9gYtW66EPacdj/mAFevznsnyoHLPy2BA8gbIQeIHoPsvwmfBftfcG//BQ== - dependencies: - original "^1.0.0" - evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz#7fcbdb198dc71959432efe13842684e0525acb02" @@ -14042,6 +14043,17 @@ extract-files@^8.0.0: resolved "https://registry.npmjs.org/extract-files/-/extract-files-8.1.0.tgz#46a0690d0fe77411a2e3804852adeaa65cd59288" integrity sha512-PTGtfthZK79WUMk+avLmwx3NGdU8+iVFXC2NMGxKsn0MnihOG2lvumj+AZo8CTwTrwjXDgZ5tztbRlEdRjBonQ== +extract-zip@2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" + integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== + dependencies: + debug "^4.1.1" + get-stream "^5.1.0" + yauzl "^2.10.0" + optionalDependencies: + "@types/yauzl" "^2.9.1" + extract-zip@^1.7.0: version "1.7.0" resolved "https://registry.npmjs.org/extract-zip/-/extract-zip-1.7.0.tgz#556cc3ae9df7f452c493a0cfb51cc30277940927" @@ -15638,6 +15650,11 @@ hash-base@^3.0.0: inherits "^2.0.1" safe-buffer "^5.0.1" +hash-it@^5.0.0: + version "5.0.2" + resolved "https://registry.npmjs.org/hash-it/-/hash-it-5.0.2.tgz#8cc981944964a5124f74f1065af0c0a5d182d556" + integrity sha512-csU3E/a9QEmEgPPxoShVuMcFWM329IGioEPRvYVBv3r5BFrU8pCfnk3jGEVvriAcwqd+nl6KsNhPPjg8MUzkhQ== + hash-stream-validation@^0.2.2: version "0.2.4" resolved "https://registry.npmjs.org/hash-stream-validation/-/hash-stream-validation-0.2.4.tgz#ee68b41bf822f7f44db1142ec28ba9ee7ccb7512" @@ -15834,13 +15851,6 @@ html-comment-regex@^1.1.0: resolved "https://registry.npmjs.org/html-comment-regex/-/html-comment-regex-1.1.2.tgz#97d4688aeb5c81886a364faa0cad1dda14d433a7" integrity sha512-P+M65QY2JQ5Y0G9KKdlDpo0zK+/OHptU5AaBwUfAIDJZk1MYf32Frm84EcOytfJE0t5JvkAnKlmjsXDnWzCJmQ== -html-encoding-sniffer@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz#e70d84b94da53aa375e11fe3a351be6642ca46f8" - integrity sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw== - dependencies: - whatwg-encoding "^1.0.1" - html-encoding-sniffer@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz#42a6dc4fd33f00281176e8b23759ca4e4fa185f3" @@ -16064,7 +16074,7 @@ https-browserify@^1.0.0: resolved "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73" integrity sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM= -https-proxy-agent@^5.0.0: +https-proxy-agent@5.0.0, https-proxy-agent@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== @@ -16109,7 +16119,7 @@ hyphenate-style-name@^1.0.2, hyphenate-style-name@^1.0.3: resolved "https://registry.npmjs.org/hyphenate-style-name/-/hyphenate-style-name-1.0.3.tgz#097bb7fa0b8f1a9cf0bd5c734cf95899981a9b48" integrity sha512-EcuixamT82oplpoJ2XU4pDtKGWQ7b00CD9f1ug9IaQ3p1bkHMiKCZ9ut9QDI6qsa6cpUuB+A/I+zLtdNK4n2DQ== -iconv-lite@0.4.24, iconv-lite@^0.4.21, iconv-lite@^0.4.24, iconv-lite@^0.4.4: +iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@^0.4.4: version "0.4.24" resolved "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== @@ -17899,38 +17909,6 @@ jscodeshift@^0.13.0: temp "^0.8.4" write-file-atomic "^2.3.0" -jsdom@11.12.0: - version "11.12.0" - resolved "https://registry.npmjs.org/jsdom/-/jsdom-11.12.0.tgz#1a80d40ddd378a1de59656e9e6dc5a3ba8657bc8" - integrity sha512-y8Px43oyiBM13Zc1z780FrfNLJCXTL40EWlty/LXUtcjykRBNgLlCjWXpfSPBl2iv+N7koQN+dvqszHZgT/Fjw== - dependencies: - abab "^2.0.0" - acorn "^5.5.3" - acorn-globals "^4.1.0" - array-equal "^1.0.0" - cssom ">= 0.3.2 < 0.4.0" - cssstyle "^1.0.0" - data-urls "^1.0.0" - domexception "^1.0.1" - escodegen "^1.9.1" - html-encoding-sniffer "^1.0.2" - left-pad "^1.3.0" - nwsapi "^2.0.7" - parse5 "4.0.0" - pn "^1.1.0" - request "^2.87.0" - request-promise-native "^1.0.5" - sax "^1.2.4" - symbol-tree "^3.2.2" - tough-cookie "^2.3.4" - w3c-hr-time "^1.0.1" - webidl-conversions "^4.0.2" - whatwg-encoding "^1.0.3" - whatwg-mimetype "^2.1.0" - whatwg-url "^6.4.1" - ws "^5.2.0" - xml-name-validator "^3.0.0" - jsdom@^16.4.0: version "16.4.0" resolved "https://registry.npmjs.org/jsdom/-/jsdom-16.4.0.tgz#36005bde2d136f73eee1a830c6d45e55408edddb" @@ -18007,6 +17985,17 @@ json-pointer@^0.6.0: dependencies: foreach "^2.0.4" +json-rules-engine@^6.1.2: + version "6.1.2" + resolved "https://registry.npmjs.org/json-rules-engine/-/json-rules-engine-6.1.2.tgz#574ef455c10973fd5de07ea8414cbb72bb84d10f" + integrity sha512-+rtKuJ33HAvFywL9broh42FA9hkZNmS0l1DmgjP7nfGJ9E2i2IsfNH0BcXjyXianp/bXAyYlsSv308AfTuvBwQ== + dependencies: + clone "^2.1.2" + eventemitter2 "^6.4.4" + hash-it "^5.0.0" + jsonpath-plus "^5.0.7" + lodash.isobjectlike "^4.0.0" + json-schema-compare@^0.2.2: version "0.2.2" resolved "https://registry.npmjs.org/json-schema-compare/-/json-schema-compare-0.2.2.tgz#dd601508335a90c7f4cfadb6b2e397225c908e56" @@ -18147,6 +18136,11 @@ jsonpath-plus@^0.19.0: resolved "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-0.19.0.tgz#b901e57607055933dc9a8bef0cc25160ee9dd64c" integrity sha512-GSVwsrzW9LsA5lzsqe4CkuZ9wp+kxBb2GwNniaWzI2YFn5Ig42rSW8ZxVpWXaAfakXNrx5pgY5AbQq7kzX29kg== +jsonpath-plus@^5.0.7: + version "5.1.0" + resolved "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-5.1.0.tgz#2fc4b2e461950626c98525425a3a3518b85af6c3" + integrity sha512-890w2Pjtj0iswAxalRlt2kHthi6HKrXEfZcn+ZNZptv7F3rUGIeDuZo+C+h4vXBHLEsVjJrHeCm35nYeZLzSBQ== + jsonpointer@^4.0.1: version "4.1.0" resolved "https://registry.npmjs.org/jsonpointer/-/jsonpointer-4.1.0.tgz#501fb89986a2389765ba09e6053299ceb4f2c2cc" @@ -18525,11 +18519,6 @@ leasot@^12.0.0: strip-ansi "^6.0.0" text-table "^0.2.0" -left-pad@^1.3.0: - version "1.3.0" - resolved "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz#5b8a3a7765dfe001261dde915589e782f8c94d1e" - integrity sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA== - lerna@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/lerna/-/lerna-4.0.0.tgz#b139d685d50ea0ca1be87713a7c2f44a5b678e9e" @@ -18931,6 +18920,11 @@ lodash.isobject@^3.0.2: resolved "https://registry.npmjs.org/lodash.isobject/-/lodash.isobject-3.0.2.tgz#3c8fb8d5b5bf4bf90ae06e14f2a530a4ed935e1d" integrity sha1-PI+41bW/S/kK4G4U8qUwpO2TXh0= +lodash.isobjectlike@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/lodash.isobjectlike/-/lodash.isobjectlike-4.0.0.tgz#742c5fc65add27924d3d24191681aa9a17b2b60d" + integrity sha1-dCxfxlrdJ5JNPSQZFoGqmheytg0= + lodash.isplainobject@^4.0.6: version "4.0.6" resolved "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb" @@ -20137,7 +20131,7 @@ mime@1.6.0, mime@^1.4.1: resolved "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== -mime@^2.2.0, mime@^2.3.1, mime@^2.4.4, mime@^2.4.6: +mime@^2.2.0, mime@^2.4.4, mime@^2.4.6: version "2.5.2" resolved "https://registry.npmjs.org/mime/-/mime-2.5.2.tgz#6e3dc6cc2b9510643830e5f19d5cb753da5eeabe" integrity sha512-tqkh47FzKeCPD2PUiPB6pkbMzsCasjxAfC62/Wap5qrUWcb+sFasXUC5I3gYM5iBM8v/Qpn4UK0x+j0iHyFPDg== @@ -20384,7 +20378,14 @@ modify-values@^1.0.0: resolved "https://registry.npmjs.org/modify-values/-/modify-values-1.0.1.tgz#b3939fa605546474e3e3e3c63d64bd43b4ee6022" integrity sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw== -moment@^2.19.3, moment@^2.27.0, moment@^2.29.1: +moment-timezone@^0.5.31: + version "0.5.33" + resolved "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.33.tgz#b252fd6bb57f341c9b59a5ab61a8e51a73bbd22c" + integrity sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w== + dependencies: + moment ">= 2.9.0" + +"moment@>= 2.9.0", moment@^2.19.3, moment@^2.27.0, moment@^2.29.1: version "2.29.1" resolved "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== @@ -20683,6 +20684,13 @@ node-cache@^5.1.2: dependencies: clone "2.x" +node-cron@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/node-cron/-/node-cron-3.0.0.tgz#b33252803e430f9cd8590cf85738efa1497a9522" + integrity sha512-DDwIvvuCwrNiaU7HEivFDULcaQualDv7KoNlB/UU1wPW0n1tDEmBJKhEIE6DlF2FuoOHcNbLJ8ITL2Iv/3AWmA== + dependencies: + moment-timezone "^0.5.31" + node-dir@^0.1.10, node-dir@^0.1.17: version "0.1.17" resolved "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz#5f5665d93351335caabef8f1c554516cf5f1e4e5" @@ -21102,7 +21110,7 @@ nunjucks@^3.2.3: asap "^2.0.3" commander "^5.1.0" -nwsapi@^2.0.7, nwsapi@^2.2.0: +nwsapi@^2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz#204879a9e3d068ff2a55139c2c772780681a38b7" integrity sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ== @@ -21365,13 +21373,6 @@ ora@^5.3.0, ora@^5.4.1: strip-ansi "^6.0.0" wcwidth "^1.0.1" -original@^1.0.0: - version "1.0.2" - resolved "https://registry.npmjs.org/original/-/original-1.0.2.tgz#e442a61cffe1c5fd20a65f3261c26663b303f25f" - integrity sha512-hyBVl6iqqUOJ8FqRe+l/gS8H+kKYjrEndd5Pm1MfBtsEKA038HkkdbAl/72EAXGyonD/PFsvmVG+EvcIpliMBg== - dependencies: - url-parse "^1.4.3" - os-browserify@^0.3.0: version "0.3.0" resolved "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz#854373c7f5c2315914fc9bfc6bd8238fdda1ec27" @@ -21810,11 +21811,6 @@ parse-url@^5.0.0: parse-path "^4.0.0" protocols "^1.4.0" -parse5@4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz#6d78656e3da8d78b4ec0b906f7c08ef1dfe3f608" - integrity sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA== - parse5@5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178" @@ -22296,6 +22292,13 @@ pirates@^4.0.0, pirates@^4.0.1: dependencies: node-modules-regexp "^1.0.0" +pkg-dir@4.2.0, pkg-dir@^4.1.0, pkg-dir@^4.2.0: + version "4.2.0" + resolved "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" + integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== + dependencies: + find-up "^4.0.0" + pkg-dir@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz#f6d5d1109e19d63edf428e0bd57e12777615334b" @@ -22310,13 +22313,6 @@ pkg-dir@^3.0.0: dependencies: find-up "^3.0.0" -pkg-dir@^4.1.0, pkg-dir@^4.2.0: - version "4.2.0" - resolved "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" - integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== - dependencies: - find-up "^4.0.0" - pkg-dir@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/pkg-dir/-/pkg-dir-5.0.0.tgz#a02d6aebe6ba133a928f74aec20bafdfe6b8e760" @@ -22348,11 +22344,6 @@ pluralize@^8.0.0: resolved "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1" integrity sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA== -pn@^1.1.0: - version "1.1.0" - resolved "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz#e2f4cef0e219f463c179ab37463e4e1ecdccbafb" - integrity sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA== - pnp-webpack-plugin@1.6.4: version "1.6.4" resolved "https://registry.npmjs.org/pnp-webpack-plugin/-/pnp-webpack-plugin-1.6.4.tgz#c9711ac4dc48a685dabafc86f8b6dd9f8df84149" @@ -22949,6 +22940,11 @@ process@^0.11.10: resolved "https://registry.npmjs.org/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" integrity sha1-czIwDoQBYb2j5podHZGn1LwW8YI= +progress@2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/progress/-/progress-2.0.1.tgz#c9242169342b1c29d275889c95734621b1952e31" + integrity sha512-OE+a6vzqazc+K6LxJrX5UPyKFvGnL5CYmq2jFGNIBWHpc4QyE49/YOumcrpQFJpfejmvRtbJzgO1zPmMCqlbBg== + progress@^2.0.0: version "2.0.3" resolved "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" @@ -23114,6 +23110,11 @@ proxy-addr@~2.0.5: forwarded "~0.1.2" ipaddr.js "1.9.1" +proxy-from-env@1.1.0: + version "1.1.0" + resolved "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + prr@~1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" @@ -23214,6 +23215,24 @@ pupa@^2.0.1: dependencies: escape-goat "^2.0.0" +puppeteer@^10.4.0: + version "10.4.0" + resolved "https://registry.npmjs.org/puppeteer/-/puppeteer-10.4.0.tgz#a6465ff97fda0576c4ac29601406f67e6fea3dc7" + integrity sha512-2cP8mBoqnu5gzAVpbZ0fRaobBWZM8GEUF4I1F6WbgHrKV/rz7SX8PG2wMymZgD0wo0UBlg2FBPNxlF/xlqW6+w== + dependencies: + debug "4.3.1" + devtools-protocol "0.0.901419" + extract-zip "2.0.1" + https-proxy-agent "5.0.0" + node-fetch "2.6.1" + pkg-dir "4.2.0" + progress "2.0.1" + proxy-from-env "1.1.0" + rimraf "3.0.2" + tar-fs "2.0.0" + unbzip2-stream "1.3.3" + ws "7.4.6" + q@^1.1.2, q@^1.5.1: version "1.5.1" resolved "https://registry.npmjs.org/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" @@ -24519,7 +24538,7 @@ request-promise-core@1.1.3: dependencies: lodash "^4.17.15" -request-promise-native@^1.0.5, request-promise-native@^1.0.8: +request-promise-native@^1.0.8: version "1.0.8" resolved "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.8.tgz#a455b960b826e44e2bf8999af64dff2bfe58cb36" integrity sha512-dapwLGqkHtwL5AEbfenuzjTYg35Jd6KPytsC2/TLkVMz8rm+tNt72MGUWT1RP/aYawMpN6HqbNGBQaRcBtjQMQ== @@ -24528,7 +24547,7 @@ request-promise-native@^1.0.5, request-promise-native@^1.0.8: stealthy-require "^1.1.1" tough-cookie "^2.3.3" -request@^2.85.0, request@^2.87.0, request@^2.88.0, request@^2.88.2: +request@^2.87.0, request@^2.88.0, request@^2.88.2: version "2.88.2" resolved "https://registry.npmjs.org/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== @@ -24746,7 +24765,7 @@ rimraf@2, rimraf@^2.2.8, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.3: dependencies: glob "^7.1.3" -rimraf@^3.0.0, rimraf@^3.0.2: +rimraf@3.0.2, rimraf@^3.0.0, rimraf@^3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== @@ -26517,7 +26536,7 @@ symbol-observable@1.2.0, symbol-observable@^1.0.4, symbol-observable@^1.1.0, sym resolved "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz#c22688aed4eab3cdc2dfeacbb561660560a00804" integrity sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ== -symbol-tree@^3.2.2, symbol-tree@^3.2.4: +symbol-tree@^3.2.4: version "3.2.4" resolved "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== @@ -26560,6 +26579,16 @@ tapable@^2.0.0, tapable@^2.1.1, tapable@^2.2.0: resolved "https://registry.npmjs.org/tapable/-/tapable-2.2.0.tgz#5c373d281d9c672848213d0e037d1c4165ab426b" integrity sha512-FBk4IesMV1rBxX2tfiK8RAmogtWn53puLOQlvO8XuwlgxcYbP4mVPS9Ph4aeamSyyVjOl24aYWAuc8U5kCVwMw== +tar-fs@2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.0.tgz#677700fc0c8b337a78bee3623fdc235f21d7afad" + integrity sha512-vaY0obB6Om/fso8a8vakQBzwholQ7v5+uy+tF3Ozvxv1KNezmVQAiWtcNmMHFSFPqL3dJA8ha6gdtFbfX9mcxA== + dependencies: + chownr "^1.1.1" + mkdirp "^0.5.1" + pump "^3.0.0" + tar-stream "^2.0.0" + tar-fs@^2.0.0, tar-fs@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" @@ -27041,7 +27070,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -tough-cookie@^2.3.3, tough-cookie@^2.3.4, tough-cookie@~2.5.0: +tough-cookie@^2.3.3, tough-cookie@~2.5.0: version "2.5.0" resolved "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== @@ -27067,13 +27096,6 @@ tough-cookie@^4.0.0: punycode "^2.1.1" universalify "^0.1.2" -tr46@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz#a8b13fd6bfd2489519674ccde55ba3693b706d09" - integrity sha1-qLE/1r/SSJUZZ0zN5VujaTtwbQk= - dependencies: - punycode "^2.1.0" - tr46@^2.0.2: version "2.0.2" resolved "https://registry.npmjs.org/tr46/-/tr46-2.0.2.tgz#03273586def1595ae08fedb38d7733cee91d2479" @@ -27502,6 +27524,14 @@ unbox-primitive@^1.0.0: has-symbols "^1.0.0" which-boxed-primitive "^1.0.1" +unbzip2-stream@1.3.3: + version "1.3.3" + resolved "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.3.3.tgz#d156d205e670d8d8c393e1c02ebd506422873f6a" + integrity sha512-fUlAF7U9Ah1Q6EieQ4x4zLNejrRvDWUYmxXUpN3uziFYCHapjWFaCAnreY9bGgxzaMCFAPPpYNng57CypwJVhg== + dependencies: + buffer "^5.2.1" + through "^2.3.8" + unc-path-regex@^0.1.2: version "0.1.2" resolved "https://registry.npmjs.org/unc-path-regex/-/unc-path-regex-0.1.2.tgz#e73dd3d7b0d7c5ed86fbac6b0ae7d8c6a69d50fa" @@ -27899,7 +27929,7 @@ url-parse-lax@^3.0.0: dependencies: prepend-http "^2.0.0" -url-parse@^1.4.3, url-parse@^1.5.3: +url-parse@^1.5.3: version "1.5.3" resolved "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz#71c1303d38fb6639ade183c2992c8cc0686df862" integrity sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ== @@ -28228,7 +28258,7 @@ vscode-languageserver-types@^3.15.1: resolved "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.15.1.tgz#17be71d78d2f6236d414f0001ce1ef4d23e6b6de" integrity sha512-+a9MPUQrNGRrGU630OGbYVQ+11iOIovjCkqxajPa9w57Sd5ruK8WQNsslzpa0x/QJqC8kRc2DUxWjIFwoNm4ZQ== -w3c-hr-time@^1.0.1, w3c-hr-time@^1.0.2: +w3c-hr-time@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd" integrity sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ== @@ -28530,7 +28560,7 @@ websocket-extensions@>=0.1.1: resolved "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz#7f8473bc839dfd87608adb95d7eb075211578a42" integrity sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg== -whatwg-encoding@^1.0.1, whatwg-encoding@^1.0.3, whatwg-encoding@^1.0.5: +whatwg-encoding@^1.0.5: version "1.0.5" resolved "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz#5abacf777c32166a51d085d6b4f3e7d27113ddb0" integrity sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw== @@ -28547,7 +28577,7 @@ whatwg-fetch@^3.4.1: resolved "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.4.1.tgz#e5f871572d6879663fa5674c8f833f15a8425ab3" integrity sha512-sofZVzE1wKwO+EYPbWfiwzaKovWiZXf4coEzjGP9b2GBVgQRLQUZ2QcuPpQExGDAW5GItpEm6Tl4OU5mywnAoQ== -whatwg-mimetype@^2.1.0, whatwg-mimetype@^2.2.0, whatwg-mimetype@^2.3.0: +whatwg-mimetype@^2.3.0: version "2.3.0" resolved "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== @@ -28819,25 +28849,16 @@ ws@7.4.5, ws@^7.4.6: resolved "https://registry.npmjs.org/ws/-/ws-7.5.5.tgz#8b4bc4af518cfabd0473ae4f99144287b33eb881" integrity sha512-BAkMFcAzl8as1G/hArkxOxq3G7pjUqQ3gzYbLL0/5zNkph70e+lCoxBGnm6AW1+/aiNeV4fnKqZ8m4GZewmH2w== -ws@^5.2.0: - version "5.2.3" - resolved "https://registry.npmjs.org/ws/-/ws-5.2.3.tgz#05541053414921bc29c63bee14b8b0dd50b07b3d" - integrity sha512-jZArVERrMsKUatIdnLzqvcfydI85dvd/Fp1u/VOpfdDWQ4c9qWXe+VIeAbQ5FrDwciAkr+lzofXLz3Kuf26AOA== - dependencies: - async-limiter "~1.0.0" +ws@7.4.6: + version "7.4.6" + resolved "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c" + integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A== "ws@^5.2.0 || ^6.0.0 || ^7.0.0", ws@^7.2.3, ws@^7.3.1: version "7.5.0" resolved "https://registry.npmjs.org/ws/-/ws-7.5.0.tgz#0033bafea031fb9df041b2026fc72a571ca44691" integrity sha512-6ezXvzOZupqKj4jUqbQ9tXuJNo+BR2gU8fFRk3XCP3e0G6WT414u5ELe6Y0vtp7kmSJ3F7YWObSNr1ESsgi4vw== -ws@^6.1.2: - version "6.2.1" - resolved "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz#442fdf0a47ed64f59b6a5d8ff130f4748ed524fb" - integrity sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA== - dependencies: - async-limiter "~1.0.0" - ws@^8.1.0: version "8.2.3" resolved "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz#63a56456db1b04367d0b721a0b80cae6d8becbba" @@ -29210,24 +29231,6 @@ zip-stream@^4.1.0: compress-commons "^4.1.0" readable-stream "^3.6.0" -zombie@^6.1.4: - version "6.1.4" - resolved "https://registry.npmjs.org/zombie/-/zombie-6.1.4.tgz#9f0f53f3d9a032beb7f3fe5b382146a3475a4d47" - integrity sha512-yxNvKtyz3PP8lkr31AYh7vdbBD4is9hYXiOQKPp+k/7GiDiFQXX1Ex+peCl4ttodu/bHZcIluJ8lxMla5XefBQ== - dependencies: - babel-runtime "6.26.0" - bluebird "^3.5.1" - debug "^4.1.0" - eventsource "^1.0.5" - iconv-lite "^0.4.21" - jsdom "11.12.0" - lodash "^4.17.10" - mime "^2.3.1" - ms "^2.1.1" - request "^2.85.0" - tough-cookie "^2.3.4" - ws "^6.1.2" - zwitch@^1.0.0: version "1.0.5" resolved "https://registry.npmjs.org/zwitch/-/zwitch-1.0.5.tgz#d11d7381ffed16b742f6af7b3f223d5cd9fe9920"