From 2b663945cb6b12e654e6d3a5c0081713f0be86c8 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 2 May 2023 19:39:03 -0400 Subject: [PATCH 01/29] fix: Minor product capitalization Signed-off-by: Adam Harvey --- plugins/techdocs/src/home/components/TechDocsPageWrapper.tsx | 2 +- plugins/techdocs/src/reader/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/techdocs/src/home/components/TechDocsPageWrapper.tsx b/plugins/techdocs/src/home/components/TechDocsPageWrapper.tsx index 52511bda19..a6192c13ee 100644 --- a/plugins/techdocs/src/home/components/TechDocsPageWrapper.tsx +++ b/plugins/techdocs/src/home/components/TechDocsPageWrapper.tsx @@ -29,7 +29,7 @@ export type TechDocsPageWrapperProps = { }; /** - * Component wrapping a techdocs page with Page and Header components + * Component wrapping a TechDocs page with Page and Header components * * @public */ diff --git a/plugins/techdocs/src/reader/README.md b/plugins/techdocs/src/reader/README.md index fe1d1ff488..58697298c6 100644 --- a/plugins/techdocs/src/reader/README.md +++ b/plugins/techdocs/src/reader/README.md @@ -2,7 +2,7 @@ The TechDocs reader is a component that fetches a remote page, runs transformers on it and renders it into a shadow dom. -Currently there's no easy way to customize which transformers to run or add new ones. If that is needed you would have to fork the techdocs plugin and make your changes in that fork. +Currently there's no easy way to customize which transformers to run or add new ones. If that is needed you would have to fork the TechDocs plugin and make your changes in that fork. Transformers are functions that optionally takes in parameters from the Reader.tsx component and returns a function which gets passed the DOM of the fetched page. A very simple transformer can look like this. From a606cae041c30ca76fc1b5f50d2c6c7693935884 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 2 May 2023 19:39:47 -0400 Subject: [PATCH 02/29] chore: Remove deprecations and use shared react imports Signed-off-by: Adam Harvey --- plugins/techdocs/src/client.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/techdocs/src/client.ts b/plugins/techdocs/src/client.ts index 499a56580c..8680e14468 100644 --- a/plugins/techdocs/src/client.ts +++ b/plugins/techdocs/src/client.ts @@ -23,11 +23,13 @@ import { } from '@backstage/core-plugin-api'; import { NotFoundError, ResponseError } from '@backstage/errors'; import { + SyncResult, + TechDocsApi, TechDocsEntityMetadata, TechDocsMetadata, + TechDocsStorageApi, } from '@backstage/plugin-techdocs-react'; import { EventSourcePolyfill } from 'event-source-polyfill'; -import { SyncResult, TechDocsApi, TechDocsStorageApi } from './api'; /** * API to talk to `techdocs-backend`. @@ -189,7 +191,7 @@ export class TechDocsStorageClient implements TechDocsStorageApi { * @param entityId - Object containing entity data like name, namespace, etc. * @param logHandler - Callback to receive log messages from the build process * @returns Whether documents are currently synchronized to newest version - * @throws Throws error on error from sync endpoint in Techdocs Backend + * @throws Throws error on error from sync endpoint in TechDocs Backend */ async syncEntityDocs( entityId: CompoundEntityRef, From 956d09e8ea6835ae47a35609fdad11486274b788 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 2 May 2023 19:45:41 -0400 Subject: [PATCH 03/29] chore: Add changeset Signed-off-by: Adam Harvey --- .changeset/short-seahorses-kiss.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/short-seahorses-kiss.md diff --git a/.changeset/short-seahorses-kiss.md b/.changeset/short-seahorses-kiss.md new file mode 100644 index 0000000000..0127e4c575 --- /dev/null +++ b/.changeset/short-seahorses-kiss.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Change deprecated local references to import from shared `plugin-techdocs-react` plugin From 40421e79563992d503a918703746964cabe611a5 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 2 May 2023 20:08:15 -0400 Subject: [PATCH 04/29] chore: Add updated API report Signed-off-by: Adam Harvey --- plugins/techdocs/api-report.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/techdocs/api-report.md b/plugins/techdocs/api-report.md index 6e14c52939..27af44bb13 100644 --- a/plugins/techdocs/api-report.md +++ b/plugins/techdocs/api-report.md @@ -20,11 +20,14 @@ import { ReactNode } from 'react'; import { ResultHighlight } from '@backstage/plugin-search-common'; import { RouteRef } from '@backstage/core-plugin-api'; import { SearchResultListItemExtensionProps } from '@backstage/plugin-search-react'; +import { SyncResult as SyncResult_2 } from '@backstage/plugin-techdocs-react'; import { TableColumn } from '@backstage/core-components'; import { TableOptions } from '@backstage/core-components'; import { TableProps } from '@backstage/core-components'; +import { TechDocsApi as TechDocsApi_2 } from '@backstage/plugin-techdocs-react'; import { TechDocsEntityMetadata as TechDocsEntityMetadata_2 } from '@backstage/plugin-techdocs-react'; import { TechDocsMetadata as TechDocsMetadata_2 } from '@backstage/plugin-techdocs-react'; +import { TechDocsStorageApi as TechDocsStorageApi_2 } from '@backstage/plugin-techdocs-react'; import { ToolbarProps } from '@material-ui/core'; import { UserListFilterKind } from '@backstage/plugin-catalog-react'; @@ -237,7 +240,7 @@ export interface TechDocsApi { export const techdocsApiRef: ApiRef; // @public -export class TechDocsClient implements TechDocsApi { +export class TechDocsClient implements TechDocsApi_2 { constructor(options: { configApi: Config; discoveryApi: DiscoveryApi; @@ -442,7 +445,7 @@ export interface TechDocsStorageApi { export const techdocsStorageApiRef: ApiRef; // @public -export class TechDocsStorageClient implements TechDocsStorageApi { +export class TechDocsStorageClient implements TechDocsStorageApi_2 { constructor(options: { configApi: Config; discoveryApi: DiscoveryApi; @@ -471,6 +474,6 @@ export class TechDocsStorageClient implements TechDocsStorageApi { syncEntityDocs( entityId: CompoundEntityRef, logHandler?: (line: string) => void, - ): Promise; + ): Promise; } ``` From 99c62d1e9cb2bb98a1a441536ab6194a894cf0ce Mon Sep 17 00:00:00 2001 From: sammy Date: Thu, 11 May 2023 16:22:59 +0100 Subject: [PATCH 05/29] docs: Add error handling instructions for 'No token available' error This commit adds instructions to the Backstage documentation on how to handle the `No token available for host: github.com, with owner , and repo from-backstage' error`. The instructions guide the user to restart Backstage using `Control-C` and `yarn dev` commands in the terminal, followed by clicking on the **START OVER** button. Signed-off-by: sammy. Signed-off-by: Samuel --- docs/getting-started/configuration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 2196a1a36d..ddf50c64e6 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -242,6 +242,7 @@ integrations: - host: github.com token: ${GITHUB_TOKEN} # this will use the environment variable GITHUB_TOKEN ``` +> If you encounter the error `No token available for host: github.com, with owner , and repo from-backstage`, try resolving it by restarting Backstage. First, stop the running instance by pressing `Control-C` in the terminal. Then, restart it by running the command `yarn dev`. Once Backstage is up and running, click the **START OVER** button." Some helpful links, for if you want to learn more about: From 4061d181c3473242f7f763fb1ce08cbce810142c Mon Sep 17 00:00:00 2001 From: Samuel Date: Thu, 11 May 2023 18:39:07 +0100 Subject: [PATCH 06/29] docs: Highlight error handling instructions with a note Signed-off-by: Samuel --- docs/getting-started/configuration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index ddf50c64e6..4147f7bf11 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -242,7 +242,8 @@ integrations: - host: github.com token: ${GITHUB_TOKEN} # this will use the environment variable GITHUB_TOKEN ``` -> If you encounter the error `No token available for host: github.com, with owner , and repo from-backstage`, try resolving it by restarting Backstage. First, stop the running instance by pressing `Control-C` in the terminal. Then, restart it by running the command `yarn dev`. Once Backstage is up and running, click the **START OVER** button." + +> Note: If you encounter the error `No token available for host: github.com, with owner , and repo from-backstage`, try resolving it by restarting Backstage. First, stop the running instance by pressing `Control-C` in the terminal. Then, restart it by running the command `yarn dev`. Once Backstage is up and running, click the **START OVER** button." Some helpful links, for if you want to learn more about: From bbf91840a52a1521cc7123187ac4b0ee7519afd9 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Sat, 18 Mar 2023 16:21:20 -0500 Subject: [PATCH 07/29] Added backend tests Signed-off-by: Andre Wanlin --- .changeset/shaggy-gorillas-occur.md | 5 + plugins/linguist-backend/api-report.md | 12 + .../20230216_remove_processed_date_default.js | 39 +++ .../src/api/LinguistBackendApi.test.ts | 309 +++++++++++++++++- .../src/api/LinguistBackendApi.ts | 17 +- .../src/db/LinguistBackendDatabase.test.ts | 169 ++++++++++ .../src/db/LinguistBackendDatabase.ts | 22 +- 7 files changed, 562 insertions(+), 11 deletions(-) create mode 100644 .changeset/shaggy-gorillas-occur.md create mode 100644 plugins/linguist-backend/migrations/20230216_remove_processed_date_default.js create mode 100644 plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts diff --git a/.changeset/shaggy-gorillas-occur.md b/.changeset/shaggy-gorillas-occur.md new file mode 100644 index 0000000000..c122032e7f --- /dev/null +++ b/.changeset/shaggy-gorillas-occur.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-linguist-backend': patch +--- + +Added tests for the `LinguistBackendDatabase` and `LinguistBackendApi` which included refactoring to better support using SQLite and removed the default from the `processes_date` column diff --git a/plugins/linguist-backend/api-report.md b/plugins/linguist-backend/api-report.md index a47f123ac4..ca36a1f816 100644 --- a/plugins/linguist-backend/api-report.md +++ b/plugins/linguist-backend/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { EntitiesOverview } from '@backstage/plugin-linguist-common'; import { EntityResults } from '@backstage/plugin-linguist-common'; import express from 'express'; import { HumanDuration } from '@backstage/types'; @@ -13,6 +14,7 @@ import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { ProcessedEntity } from '@backstage/plugin-linguist-common'; +import { Results } from 'linguist-js/dist/types'; import { TaskScheduleDefinition } from '@backstage/backend-tasks'; import { TokenManager } from '@backstage/backend-common'; import { UrlReader } from '@backstage/backend-common'; @@ -38,8 +40,18 @@ export class LinguistBackendApi { linguistJsOptions?: Record, ); // (undocumented) + addNewEntities(): Promise; + // (undocumented) + generateEntitiesLanguages(): Promise; + // (undocumented) + generateEntityLanguages(entityRef: string, url: string): Promise; + // (undocumented) + getEntitiesOverview(): Promise; + // (undocumented) getEntityLanguages(entityRef: string): Promise; // (undocumented) + getLinguistResults(dir: string): Promise; + // (undocumented) processEntities(): Promise; } diff --git a/plugins/linguist-backend/migrations/20230216_remove_processed_date_default.js b/plugins/linguist-backend/migrations/20230216_remove_processed_date_default.js new file mode 100644 index 0000000000..5aaf02216e --- /dev/null +++ b/plugins/linguist-backend/migrations/20230216_remove_processed_date_default.js @@ -0,0 +1,39 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + // Sqlite does not support this raw SQL + if (!knex.client.config.client.includes('sqlite3')) { + await knex.raw( + 'ALTER TABLE entity_result ALTER COLUMN processed_date DROP DEFAULT;', + ); + } +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + // Sqlite does not support this raw SQL + if (!knex.client.config.client.includes('sqlite3')) { + await knex.raw( + 'ALTER TABLE entity_result ALTER COLUMN processed_date SET DEFAULT now();', + ); + } +}; diff --git a/plugins/linguist-backend/src/api/LinguistBackendApi.test.ts b/plugins/linguist-backend/src/api/LinguistBackendApi.test.ts index 741d52e734..baf7d27ad7 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendApi.test.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendApi.test.ts @@ -13,7 +13,49 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { kindOrDefault } from './LinguistBackendApi'; +import { + getVoidLogger, + PluginEndpointDiscovery, + ReadTreeResponse, + ServerTokenManager, + UrlReader, +} from '@backstage/backend-common'; +import { GetEntitiesResponse } from '@backstage/catalog-client'; +import { Results } from 'linguist-js/dist/types'; +import { DateTime } from 'luxon'; +import { LinguistBackendStore } from '../db'; +import { kindOrDefault, LinguistBackendApi } from './LinguistBackendApi'; +import fs from 'fs-extra'; + +const linguistResultMock = Promise.resolve({ + files: { + count: 4, + bytes: 6010, + results: { + '/src/index.ts': 'TypeScript', + '/src/cli.js': 'JavaScript', + '/readme.md': 'Markdown', + '/no-lang': null, + }, + }, + languages: { + count: 3, + bytes: 6000, + results: { + JavaScript: { type: 'programming', bytes: 1000, color: '#f1e05a' }, + TypeScript: { type: 'programming', bytes: 2000, color: '#2b7489' }, + Markdown: { type: 'prose', bytes: 3000, color: '#083fa1' }, + }, + }, + unknown: { + count: 1, + bytes: 10, + filenames: { + 'no-lang': 10, + }, + extensions: {}, + }, +} as Results); describe('kindOrDefault', () => { it('should return default kind when undefined', () => { @@ -26,3 +68,268 @@ describe('kindOrDefault', () => { expect(kindOrDefault(['API'])).toEqual(['API']); }); }); + +describe('Linguist backend API', () => { + const getEntitiesMock = jest.fn(); + jest.mock('@backstage/catalog-client', () => { + return { + CatalogClient: jest + .fn() + .mockImplementation(() => ({ getEntities: getEntitiesMock })), + }; + }); + + const logger = getVoidLogger(); + + const store: jest.Mocked = { + insertEntityResults: jest.fn(), + insertNewEntity: jest.fn(), + getEntityResults: jest.fn(), + getProcessedEntities: jest.fn(), + getUnprocessedEntities: jest.fn(), + }; + + const urlReader: jest.Mocked = { + readTree: jest.fn(), + search: jest.fn(), + readUrl: jest.fn(), + }; + + const discovery: jest.Mocked = { + getBaseUrl: jest.fn(), + getExternalBaseUrl: jest.fn(), + }; + + const tokenManager = ServerTokenManager.noop(); + + const api = new LinguistBackendApi( + logger, + store, + urlReader, + discovery, + tokenManager, + ); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should get languages for an entity', async () => { + store.getEntityResults.mockResolvedValue({ + languageCount: 1, + totalBytes: 2205, + processedDate: '2023-02-15T20:10:21.378Z', + breakdown: [ + { + name: 'YAML', + percentage: 100, + bytes: 2205, + type: 'data', + color: '#cb171e', + }, + ], + }); + + const entityRef = 'template:default/create-react-app-template'; + const languages = await api.getEntityLanguages(entityRef); + expect(languages).toEqual({ + languageCount: 1, + totalBytes: 2205, + processedDate: '2023-02-15T20:10:21.378Z', + breakdown: [ + { + name: 'YAML', + percentage: 100, + bytes: 2205, + type: 'data', + color: '#cb171e', + }, + ], + }); + }); + + it('should add new entities', async () => { + const testEntityListResponse: GetEntitiesResponse = { + items: [ + { + apiVersion: 'backstage.io/v1beta1', + metadata: { + name: 'service-one', + }, + kind: 'Component', + }, + { + apiVersion: 'backstage.io/v1beta1', + metadata: { + name: 'service-two', + }, + kind: 'Component', + }, + { + apiVersion: 'backstage.io/v1beta1', + metadata: { + name: 'service-three', + }, + kind: 'Component', + }, + ], + }; + getEntitiesMock.mockResolvedValue(testEntityListResponse); + + await api.addNewEntities(); + expect(store.insertNewEntity).toHaveBeenCalledTimes(3); + }); + + it('should get default entity overview', async () => { + store.getProcessedEntities.mockResolvedValue([ + { + entityRef: 'component:default/service-one', + processedDate: DateTime.now().toJSDate(), + }, + { + entityRef: 'component:default/stale-service-two', + processedDate: DateTime.now().minus({ days: 45 }).toJSDate(), + }, + ]); + + store.getUnprocessedEntities.mockResolvedValue([ + 'component:default/service-three', + 'component:default/service-four', + 'component:default/service-five', + ]); + + const overview = await api.getEntitiesOverview(); + expect(overview.entityCount).toEqual(5); + expect(overview.processedCount).toEqual(2); + expect(overview.staleCount).toEqual(0); + expect(overview.pendingCount).toEqual(3); + expect(overview.filteredEntities).toEqual([ + 'component:default/service-three', + 'component:default/service-four', + 'component:default/service-five', + ]); + }); + + it('should get entity overview with stale items', async () => { + const staleApi = new LinguistBackendApi( + logger, + store, + urlReader, + discovery, + tokenManager, + { days: 5 }, + ); + store.getProcessedEntities.mockResolvedValue([ + { + entityRef: 'component:default/service-one', + processedDate: DateTime.now().toJSDate(), + }, + { + entityRef: 'component:default/stale-service-two', + processedDate: DateTime.now().minus({ days: 45 }).toJSDate(), + }, + ]); + + store.getUnprocessedEntities.mockResolvedValue([ + 'component:default/service-three', + 'component:default/service-four', + 'component:default/service-five', + ]); + + const overview = await staleApi.getEntitiesOverview(); + expect(overview.entityCount).toEqual(5); + expect(overview.processedCount).toEqual(2); + expect(overview.staleCount).toEqual(1); + expect(overview.pendingCount).toEqual(4); + expect(overview.filteredEntities).toEqual([ + 'component:default/stale-service-two', + 'component:default/service-three', + 'component:default/service-four', + 'component:default/service-five', + ]); + }); + + it('should generate and save languages for an entity', async () => { + const spy = jest + .spyOn(api, 'getLinguistResults') + .mockImplementation(() => linguistResultMock); + + urlReader.readTree.mockResolvedValueOnce({ + files: async () => [ + { + content: async () => Buffer.from('-- XXX: code-data', 'utf8'), + path: 'my-file.js', + }, + ], + dir: async () => '/temp/my-code', + } as ReadTreeResponse); + + const fsSpy = jest.spyOn(fs, 'remove'); + + await api.generateEntityLanguages( + 'component:default/fake-service', + 'https://some.fake/service/', + ); + expect(api.getLinguistResults).toHaveBeenCalled(); + expect(store.insertEntityResults).toHaveBeenCalled(); + expect(fs.remove).toHaveBeenCalled(); + spy.mockClear(); + fsSpy.mockClear(); + }); + + it('should generate languages for multiple entities using default', async () => { + store.getProcessedEntities.mockResolvedValue([ + { + entityRef: 'component:default/service-one', + processedDate: DateTime.now().toJSDate(), + }, + { + entityRef: 'component:default/stale-service-two', + processedDate: DateTime.now().minus({ days: 45 }).toJSDate(), + }, + ]); + + store.getUnprocessedEntities.mockResolvedValue([ + 'component:default/service-three', + 'component:default/service-four', + 'component:default/service-five', + ]); + const generateEntityLanguages = jest.spyOn(api, 'generateEntityLanguages'); + await api.generateEntitiesLanguages(); + expect(generateEntityLanguages).toHaveBeenCalledTimes(3); + }); + + it('should generate languages for multiple entities using defined batch size', async () => { + const batchApi = new LinguistBackendApi( + logger, + store, + urlReader, + discovery, + tokenManager, + undefined, + 1, + ); + store.getProcessedEntities.mockResolvedValue([ + { + entityRef: 'component:default/service-one', + processedDate: DateTime.now().toJSDate(), + }, + { + entityRef: 'component:default/stale-service-two', + processedDate: DateTime.now().minus({ days: 45 }).toJSDate(), + }, + ]); + + store.getUnprocessedEntities.mockResolvedValue([ + 'component:default/service-three', + 'component:default/service-four', + 'component:default/service-five', + ]); + const generateEntityLanguages = jest.spyOn( + batchApi, + 'generateEntityLanguages', + ); + await batchApi.generateEntitiesLanguages(); + expect(generateEntityLanguages).toHaveBeenCalledTimes(1); + }); +}); diff --git a/plugins/linguist-backend/src/api/LinguistBackendApi.ts b/plugins/linguist-backend/src/api/LinguistBackendApi.ts index df7e30a7b0..766012018f 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendApi.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendApi.ts @@ -99,7 +99,7 @@ export class LinguistBackendApi { await this.generateEntitiesLanguages(); } - private async addNewEntities() { + public async addNewEntities() { const annotationKey = this.useSourceLocation ? ANNOTATION_SOURCE_LOCATION : LINGUIST_ANNOTATION; @@ -121,7 +121,7 @@ export class LinguistBackendApi { }); } - private async generateEntitiesLanguages() { + public async generateEntitiesLanguages() { const entitiesOverview = await this.getEntitiesOverview(); this.logger?.info( `Entities overview: Entity: ${entitiesOverview.entityCount}, Processed: ${entitiesOverview.processedCount}, Pending: ${entitiesOverview.pendingCount}, Stale ${entitiesOverview.staleCount}`, @@ -156,7 +156,7 @@ export class LinguistBackendApi { }); } - private async getEntitiesOverview(): Promise { + public async getEntitiesOverview(): Promise { this.logger?.debug('Getting pending entities'); const processedEntities = await this.store.getProcessedEntities(); @@ -172,7 +172,7 @@ export class LinguistBackendApi { const filteredEntities = staleEntities.concat(unprocessedEntities); const entitiesOverview: EntitiesOverview = { - entityCount: unprocessedEntities.length, + entityCount: unprocessedEntities.length + processedEntities.length, processedCount: processedEntities.length, staleCount: staleEntities.length, pendingCount: filteredEntities.length, @@ -182,7 +182,7 @@ export class LinguistBackendApi { return entitiesOverview; } - private async generateEntityLanguages( + public async generateEntityLanguages( entityRef: string, url: string, ): Promise { @@ -193,7 +193,7 @@ export class LinguistBackendApi { const readTreeResponse = await this.urlReader.readTree(url); const dir = await readTreeResponse.dir(); - const results = await linguist(dir, this.linguistJsOptions); + const results = await this.getLinguistResults(dir); try { const totalBytes = results.languages.bytes; @@ -233,6 +233,11 @@ export class LinguistBackendApi { await fs.remove(dir); } } + + public async getLinguistResults(dir: string) { + const results = await linguist(dir, this.linguistJsOptions); + return results; + } } export function kindOrDefault(kind?: string[]) { diff --git a/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts b/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts new file mode 100644 index 0000000000..f4e0bba627 --- /dev/null +++ b/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts @@ -0,0 +1,169 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex as KnexType, Knex } from 'knex'; +import { TestDatabases } from '@backstage/backend-test-utils'; +import { + LinguistBackendDatabase, + LinguistBackendStore, +} from './LinguistBackendDatabase'; +import { Languages, ProcessedEntity } from '@backstage/plugin-linguist-common'; + +function createDatabaseManager( + client: KnexType, + skipMigrations: boolean = false, +) { + return { + getClient: async () => client, + migrations: { + skip: skipMigrations, + }, + }; +} + +const rawDbEntityResultRows = [ + { + id: '14b32439-848a-49b2-92a4-98753f932606', + entity_ref: 'template:default/create-react-app-template', + languages: undefined, + processed_date: undefined, + }, + { + id: '8c85d3ec-ccea-4b29-9de9-ccdd7e5ba452', + entity_ref: 'template:default/docs-template', + languages: undefined, + processed_date: undefined, + }, + { + id: 'b922c87b-37bc-4505-b4af-70a1438decda', + entity_ref: 'template:default/pull-request', + languages: + '{"languageCount":1,"totalBytes":2205,"processedDate":"2023-02-15T20:10:21.378Z","breakdown":[{"name":"YAML","percentage":100,"bytes":2205,"type":"data","color":"#cb171e"}]}', + processed_date: new Date('2023-02-15 20:10:21.378Z'), + }, + { + id: 'bd555e6d-a3d0-4b48-a930-194db8f80db7', + entity_ref: 'template:default/react-ssr-template', + languages: + '{"languageCount":8,"totalBytes":8988,"processedDate":"2023-02-15T20:10:21.388Z","breakdown":[{"name":"INI","percentage":2.26,"bytes":203,"type":"data","color":"#d1dbe0"},{"name":"JavaScript","percentage":5.94,"bytes":534,"type":"programming","color":"#f1e05a"},{"name":"YAML","percentage":31.09,"bytes":2794,"type":"data","color":"#cb171e"},{"name":"Markdown","percentage":11.79,"bytes":1060,"type":"prose","color":"#083fa1"},{"name":"JSON","percentage":21.09,"bytes":1896,"type":"data","color":"#292929"},{"name":"CSS","percentage":0,"bytes":0,"type":"markup","color":"#563d7c"},{"name":"TSX","percentage":26.01,"bytes":2338,"type":"programming","color":"#3178c6"},{"name":"TypeScript","percentage":1.81,"bytes":163,"type":"programming","color":"#3178c6"}]}', + processed_date: new Date('2023-02-15 20:10:21.388Z'), + }, + { + id: '4145c0cf-44e9-4e95-9d57-781af4685b28', + entity_ref: 'template:default/springboot-template', + languages: + '{"languageCount":9,"totalBytes":80986,"processedDate":"2023-02-15T20:10:21.419Z","breakdown":[{"name":"Shell","percentage":0.16,"bytes":130,"type":"programming","color":"#89e051"},{"name":"INI","percentage":0.35,"bytes":286,"type":"data","color":"#d1dbe0"},{"name":"Dockerfile","percentage":0.3,"bytes":246,"type":"programming","color":"#384d54"},{"name":"YAML","percentage":3.92,"bytes":3171,"type":"data","color":"#cb171e"},{"name":"Markdown","percentage":1.31,"bytes":1059,"type":"prose","color":"#083fa1"},{"name":"XML","percentage":10.48,"bytes":8491,"type":"data","color":"#0060ac"},{"name":"Java","percentage":1.8,"bytes":1455,"type":"programming","color":"#b07219"},{"name":"Text","percentage":81.22,"bytes":65780,"type":"prose"},{"name":"Protocol Buffer","percentage":0.45,"bytes":368,"type":"data"}]}', + processed_date: new Date('2023-02-15 20:10:21.419Z'), + }, +]; + +describe('Linguist database', () => { + const databases = TestDatabases.create(); + let store: LinguistBackendStore; + let testDbClient: Knex; + beforeAll(async () => { + testDbClient = await databases.init('SQLITE_3'); + const database = createDatabaseManager(testDbClient); + + store = await LinguistBackendDatabase.create(await database.getClient()); + }); + beforeEach(async () => { + await testDbClient.batchInsert('entity_result', rawDbEntityResultRows); + }); + afterEach(async () => { + await testDbClient('entity_result').delete(); + }); + + it('should be able to return entity results', async () => { + const validLanguagesResult: Languages = { + languageCount: 1, + totalBytes: 2205, + processedDate: '2023-02-15T20:10:21.378Z', + breakdown: [ + { + name: 'YAML', + percentage: 100, + bytes: 2205, + type: 'data', + color: '#cb171e', + }, + ], + }; + + const entityResult = await store.getEntityResults( + 'template:default/pull-request', + ); + expect(entityResult).toMatchObject(validLanguagesResult); + }); + + it('should return empty entity results when not found', async () => { + const validEmptyLanguagesResult: Languages = { + languageCount: 0, + totalBytes: 0, + processedDate: 'undefined', + breakdown: [], + }; + + const entityResult = await store.getEntityResults( + 'template:default/create-react-app-template', + ); + expect(entityResult).toMatchObject(validEmptyLanguagesResult); + }); + + it('should be able to return unprocessed entities', async () => { + const validUnprocessedEntities: string[] = [ + 'template:default/create-react-app-template', + 'template:default/docs-template', + ]; + + const unprocessedEntities = await store.getUnprocessedEntities(); + + expect(unprocessedEntities).toMatchObject(validUnprocessedEntities); + }); + + it('should be able to return processed entities', async () => { + const validProcessedEntities: ProcessedEntity[] = [ + { + entityRef: 'template:default/pull-request', + processedDate: new Date('2023-02-15 20:10:21.378Z'), + }, + { + entityRef: 'template:default/react-ssr-template', + processedDate: new Date('2023-02-15 20:10:21.388Z'), + }, + { + entityRef: 'template:default/springboot-template', + processedDate: new Date('2023-02-15 20:10:21.419Z'), + }, + ]; + + const processedEntities = await store.getProcessedEntities(); + + expect(processedEntities).toMatchObject(validProcessedEntities); + }); + + it('should insert new entities and ignore duplicates', async () => { + const before = testDbClient.count('entity_result'); + + await store.insertNewEntity('component:/default/new-entity-one'); + await store.insertNewEntity('component:/default/new-entity-two'); + await store.insertNewEntity('template:default/pull-request'); + + const after = testDbClient.count('entity_result'); + + expect(before).toEqual(after); + }); +}); diff --git a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts index cf93374800..c045d0a4a9 100644 --- a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts +++ b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts @@ -26,8 +26,8 @@ import { export type RawDbEntityResultRow = { id: string; entity_ref: string; - languages: string; - processed_date: Date; + languages?: string; + processed_date?: Date; }; /** @public */ @@ -102,7 +102,7 @@ export class LinguistBackendDatabase implements LinguistBackendStore { } try { - return JSON.parse(entityResults.languages); + return JSON.parse(entityResults.languages as string); } catch (error) { throw new Error(`Failed to parse languages for '${entityRef}', ${error}`); } @@ -118,9 +118,20 @@ export class LinguistBackendDatabase implements LinguistBackendStore { } const processedEntities = rawEntities.map(rawEntity => { + // Note: processed_date should never be null, this is handled by the DB query above + let processedDate = new Date(); + if (rawEntity.processed_date) { + // SQLite will return a Timestamp whereas Postgres will return a proper Date + // This tests to see if we are getting a timestamp and convert if needed + processedDate = new Date(+rawEntity.processed_date.toString()); + if (isNaN(+rawEntity.processed_date.toString())) { + processedDate = rawEntity.processed_date; + } + } + const processEntity = { entityRef: rawEntity.entity_ref, - processedDate: rawEntity.processed_date, + processedDate: processedDate, }; return processEntity; @@ -131,6 +142,9 @@ export class LinguistBackendDatabase implements LinguistBackendStore { async getUnprocessedEntities(): Promise { const rawEntities = await this.db('entity_result') + // TODO(ahhhndre) processed_date should always be null as well but it had a default to the current date + // once the default has been removed and released, we can then come back an enable this check + // .whereNull('processed_date') .whereNull('languages') .orderBy('created_at', 'asc'); From 72b27b1bf6fb37023b86fe31815f4459660e9902 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Sat, 18 Mar 2023 17:09:04 -0500 Subject: [PATCH 08/29] Added backend-test-utils Signed-off-by: Andre Wanlin --- plugins/linguist-backend/package.json | 1 + yarn.lock | 1 + 2 files changed, 2 insertions(+) diff --git a/plugins/linguist-backend/package.json b/plugins/linguist-backend/package.json index f3feacb740..a559ec5222 100644 --- a/plugins/linguist-backend/package.json +++ b/plugins/linguist-backend/package.json @@ -44,6 +44,7 @@ "yn": "^4.0.0" }, "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", "@types/supertest": "^2.0.8", "msw": "^1.0.0", diff --git a/yarn.lock b/yarn.lock index 977fcbf506..d12796c4e5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7573,6 +7573,7 @@ __metadata: dependencies: "@backstage/backend-common": "workspace:^" "@backstage/backend-tasks": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" "@backstage/catalog-client": "workspace:^" "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" From 23fade04fd460d72470b3cc7f4482f8bf73dc198 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Sun, 26 Mar 2023 13:25:07 -0500 Subject: [PATCH 09/29] Completed tests Signed-off-by: Andre Wanlin --- plugins/linguist-backend/api-report.md | 3 +- .../src/api/LinguistBackendApi.test.ts | 117 +++++++++++++----- .../src/api/LinguistBackendApi.ts | 26 ++-- .../linguist-backend/src/service/router.ts | 5 +- 4 files changed, 106 insertions(+), 45 deletions(-) diff --git a/plugins/linguist-backend/api-report.md b/plugins/linguist-backend/api-report.md index ca36a1f816..907a48c50f 100644 --- a/plugins/linguist-backend/api-report.md +++ b/plugins/linguist-backend/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { CatalogApi } from '@backstage/catalog-client'; import { EntitiesOverview } from '@backstage/plugin-linguist-common'; import { EntityResults } from '@backstage/plugin-linguist-common'; import express from 'express'; @@ -31,8 +32,8 @@ export class LinguistBackendApi { logger: Logger, store: LinguistBackendStore, urlReader: UrlReader, - discovery: PluginEndpointDiscovery, tokenManager: TokenManager, + catalogApi: CatalogApi, age?: HumanDuration, batchSize?: number, useSourceLocation?: boolean, diff --git a/plugins/linguist-backend/src/api/LinguistBackendApi.test.ts b/plugins/linguist-backend/src/api/LinguistBackendApi.test.ts index baf7d27ad7..b0f20f0207 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendApi.test.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendApi.test.ts @@ -13,19 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger, - PluginEndpointDiscovery, ReadTreeResponse, ServerTokenManager, UrlReader, } from '@backstage/backend-common'; -import { GetEntitiesResponse } from '@backstage/catalog-client'; +import { CatalogApi, GetEntitiesResponse } from '@backstage/catalog-client'; import { Results } from 'linguist-js/dist/types'; import { DateTime } from 'luxon'; import { LinguistBackendStore } from '../db'; import { kindOrDefault, LinguistBackendApi } from './LinguistBackendApi'; import fs from 'fs-extra'; +import { LINGUIST_ANNOTATION } from '@backstage/plugin-linguist-common'; const linguistResultMock = Promise.resolve({ files: { @@ -70,15 +71,6 @@ describe('kindOrDefault', () => { }); describe('Linguist backend API', () => { - const getEntitiesMock = jest.fn(); - jest.mock('@backstage/catalog-client', () => { - return { - CatalogClient: jest - .fn() - .mockImplementation(() => ({ getEntities: getEntitiesMock })), - }; - }); - const logger = getVoidLogger(); const store: jest.Mocked = { @@ -95,10 +87,10 @@ describe('Linguist backend API', () => { readUrl: jest.fn(), }; - const discovery: jest.Mocked = { - getBaseUrl: jest.fn(), - getExternalBaseUrl: jest.fn(), - }; + const catalogApi: jest.Mocked = { + getEntities: jest.fn(), + getEntityByRef: jest.fn(), + } as any; const tokenManager = ServerTokenManager.noop(); @@ -106,8 +98,8 @@ describe('Linguist backend API', () => { logger, store, urlReader, - discovery, tokenManager, + catalogApi, ); beforeEach(() => { @@ -174,7 +166,7 @@ describe('Linguist backend API', () => { }, ], }; - getEntitiesMock.mockResolvedValue(testEntityListResponse); + catalogApi.getEntities.mockResolvedValue(testEntityListResponse); await api.addNewEntities(); expect(store.insertNewEntity).toHaveBeenCalledTimes(3); @@ -211,12 +203,12 @@ describe('Linguist backend API', () => { }); it('should get entity overview with stale items', async () => { - const staleApi = new LinguistBackendApi( + const apiWithAge = new LinguistBackendApi( logger, store, urlReader, - discovery, tokenManager, + catalogApi, { days: 5 }, ); store.getProcessedEntities.mockResolvedValue([ @@ -236,7 +228,7 @@ describe('Linguist backend API', () => { 'component:default/service-five', ]); - const overview = await staleApi.getEntitiesOverview(); + const overview = await apiWithAge.getEntitiesOverview(); expect(overview.entityCount).toEqual(5); expect(overview.processedCount).toEqual(2); expect(overview.staleCount).toEqual(1); @@ -277,7 +269,7 @@ describe('Linguist backend API', () => { fsSpy.mockClear(); }); - it('should generate languages for multiple entities using default', async () => { + it('should generate languages for entities using default', async () => { store.getProcessedEntities.mockResolvedValue([ { entityRef: 'component:default/service-one', @@ -294,21 +286,56 @@ describe('Linguist backend API', () => { 'component:default/service-four', 'component:default/service-five', ]); + + const entity = { + apiVersion: 'backstage.io/v1beta1', + metadata: { + name: 'service-one', + annotations: { + [LINGUIST_ANNOTATION]: 'https://some.fake/service/', + }, + }, + kind: 'Component', + }; + + catalogApi.getEntityByRef.mockResolvedValue(entity); + + const resultsSpy = jest + .spyOn(api, 'getLinguistResults') + .mockImplementation(() => linguistResultMock); + + urlReader.readTree.mockResolvedValue({ + files: async () => [ + { + content: async () => Buffer.from('-- XXX: code-data', 'utf8'), + path: 'my-file.js', + }, + ], + dir: async () => '/temp/my-code', + } as ReadTreeResponse); + + const fsSpy = jest.spyOn(fs, 'remove'); + const generateEntityLanguages = jest.spyOn(api, 'generateEntityLanguages'); await api.generateEntitiesLanguages(); expect(generateEntityLanguages).toHaveBeenCalledTimes(3); + + generateEntityLanguages.mockClear(); + resultsSpy.mockClear(); + fsSpy.mockClear(); }); - it('should generate languages for multiple entities using defined batch size', async () => { - const batchApi = new LinguistBackendApi( + it('should generate languages for entities using defined batch size', async () => { + const apiWithBatchSize = new LinguistBackendApi( logger, store, urlReader, - discovery, tokenManager, + catalogApi, undefined, - 1, + 2, ); + store.getProcessedEntities.mockResolvedValue([ { entityRef: 'component:default/service-one', @@ -325,11 +352,45 @@ describe('Linguist backend API', () => { 'component:default/service-four', 'component:default/service-five', ]); + + const entity = { + apiVersion: 'backstage.io/v1beta1', + metadata: { + name: 'service-one', + annotations: { + [LINGUIST_ANNOTATION]: 'https://some.fake/service/', + }, + }, + kind: 'Component', + }; + + catalogApi.getEntityByRef.mockResolvedValue(entity); + + const resultsSpy = jest + .spyOn(apiWithBatchSize, 'getLinguistResults') + .mockImplementation(() => linguistResultMock); + + urlReader.readTree.mockResolvedValue({ + files: async () => [ + { + content: async () => Buffer.from('-- XXX: code-data', 'utf8'), + path: 'my-file.js', + }, + ], + dir: async () => '/temp/my-code', + } as ReadTreeResponse); + + const fsSpy = jest.spyOn(fs, 'remove'); + const generateEntityLanguages = jest.spyOn( - batchApi, + apiWithBatchSize, 'generateEntityLanguages', ); - await batchApi.generateEntitiesLanguages(); - expect(generateEntityLanguages).toHaveBeenCalledTimes(1); + await apiWithBatchSize.generateEntitiesLanguages(); + expect(generateEntityLanguages).toHaveBeenCalledTimes(2); + + generateEntityLanguages.mockClear(); + resultsSpy.mockClear(); + fsSpy.mockClear(); }); }); diff --git a/plugins/linguist-backend/src/api/LinguistBackendApi.ts b/plugins/linguist-backend/src/api/LinguistBackendApi.ts index 766012018f..f00aebcee1 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendApi.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendApi.ts @@ -22,14 +22,10 @@ import { } from '@backstage/plugin-linguist-common'; import { CATALOG_FILTER_EXISTS, - CatalogClient, GetEntitiesRequest, + CatalogApi, } from '@backstage/catalog-client'; -import { - PluginEndpointDiscovery, - TokenManager, - UrlReader, -} from '@backstage/backend-common'; +import { TokenManager, UrlReader } from '@backstage/backend-common'; import { DateTime } from 'luxon'; import { LINGUIST_ANNOTATION } from '@backstage/plugin-linguist-common'; @@ -49,10 +45,9 @@ export class LinguistBackendApi { private readonly logger: Logger; private readonly store: LinguistBackendStore; private readonly urlReader: UrlReader; - private readonly discovery: PluginEndpointDiscovery; private readonly tokenManager: TokenManager; - private readonly catalogClient: CatalogClient; + private readonly catalogApi: CatalogApi; private readonly age?: HumanDuration; private readonly batchSize?: number; private readonly useSourceLocation?: boolean; @@ -62,8 +57,8 @@ export class LinguistBackendApi { logger: Logger, store: LinguistBackendStore, urlReader: UrlReader, - discovery: PluginEndpointDiscovery, tokenManager: TokenManager, + catalogApi: CatalogApi, age?: HumanDuration, batchSize?: number, useSourceLocation?: boolean, @@ -73,9 +68,8 @@ export class LinguistBackendApi { this.logger = logger; this.store = store; this.urlReader = urlReader; - this.discovery = discovery; this.tokenManager = tokenManager; - this.catalogClient = new CatalogClient({ discoveryApi: this.discovery }); + this.catalogApi = catalogApi; this.batchSize = batchSize; this.age = age; this.useSourceLocation = useSourceLocation; @@ -112,7 +106,7 @@ export class LinguistBackendApi { }; const { token } = await this.tokenManager.getToken(); - const response = await this.catalogClient.getEntities(request, { token }); + const response = await this.catalogApi.getEntities(request, { token }); const entities = response.items; entities.forEach(entity => { @@ -131,9 +125,10 @@ export class LinguistBackendApi { 0, this.batchSize ?? 20, ); - entities.forEach(async entityRef => { + + for (const entityRef of entities) { const { token } = await this.tokenManager.getToken(); - const entity = await this.catalogClient.getEntityByRef(entityRef, { + const entity = await this.catalogApi.getEntityByRef(entityRef, { token, }); const annotationKey = this.useSourceLocation @@ -148,12 +143,13 @@ export class LinguistBackendApi { try { await this.generateEntityLanguages(entityRef, url); } catch (error) { + console.log(error); assertError(error); this.logger.error( `Unable to process "${entityRef}" using "${url}", message: ${error.message}, stack: ${error.stack}`, ); } - }); + } } public async getEntitiesOverview(): Promise { diff --git a/plugins/linguist-backend/src/service/router.ts b/plugins/linguist-backend/src/service/router.ts index 8d52a2e01d..4c636aa854 100644 --- a/plugins/linguist-backend/src/service/router.ts +++ b/plugins/linguist-backend/src/service/router.ts @@ -31,6 +31,7 @@ import { TaskScheduleDefinition, } from '@backstage/backend-tasks'; import { HumanDuration } from '@backstage/types'; +import { CatalogClient } from '@backstage/catalog-client'; /** @public */ export interface PluginOptions { @@ -74,14 +75,16 @@ export async function createRouter( await database.getClient(), ); + const catalogClient = new CatalogClient({ discoveryApi: discovery }); + const linguistBackendApi = routerOptions.linguistBackendApi || new LinguistBackendApi( logger, linguistBackendStore, reader, - discovery, tokenManager, + catalogClient, age, batchSize, useSourceLocation, From 4e08dda90960a5618c1ef7965497694fe8a03391 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Fri, 7 Apr 2023 08:07:39 -0500 Subject: [PATCH 10/29] Implemented interface for API Signed-off-by: Andre Wanlin --- plugins/linguist-backend/api-report.md | 22 ++++++-------- ....test.ts => LinguistBackendClient.test.ts} | 8 ++--- ...BackendApi.ts => LinguistBackendClient.ts} | 30 +++++++++++++------ plugins/linguist-backend/src/api/index.ts | 3 +- plugins/linguist-backend/src/index.ts | 3 +- .../src/service/router.test.ts | 2 +- .../linguist-backend/src/service/router.ts | 10 +++---- 7 files changed, 44 insertions(+), 34 deletions(-) rename plugins/linguist-backend/src/api/{LinguistBackendApi.test.ts => LinguistBackendClient.test.ts} (97%) rename plugins/linguist-backend/src/api/{LinguistBackendApi.ts => LinguistBackendClient.ts} (90%) diff --git a/plugins/linguist-backend/api-report.md b/plugins/linguist-backend/api-report.md index 907a48c50f..795271f46c 100644 --- a/plugins/linguist-backend/api-report.md +++ b/plugins/linguist-backend/api-report.md @@ -4,7 +4,6 @@ ```ts import { CatalogApi } from '@backstage/catalog-client'; -import { EntitiesOverview } from '@backstage/plugin-linguist-common'; import { EntityResults } from '@backstage/plugin-linguist-common'; import express from 'express'; import { HumanDuration } from '@backstage/types'; @@ -15,7 +14,6 @@ import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { ProcessedEntity } from '@backstage/plugin-linguist-common'; -import { Results } from 'linguist-js/dist/types'; import { TaskScheduleDefinition } from '@backstage/backend-tasks'; import { TokenManager } from '@backstage/backend-common'; import { UrlReader } from '@backstage/backend-common'; @@ -27,7 +25,15 @@ export function createRouter( ): Promise; // @public (undocumented) -export class LinguistBackendApi { +export interface LinguistBackendApi { + // (undocumented) + getEntityLanguages(entityRef: string): Promise; + // (undocumented) + processEntities(): Promise; +} + +// @public (undocumented) +export class LinguistBackendClient implements LinguistBackendApi { constructor( logger: Logger, store: LinguistBackendStore, @@ -41,18 +47,8 @@ export class LinguistBackendApi { linguistJsOptions?: Record, ); // (undocumented) - addNewEntities(): Promise; - // (undocumented) - generateEntitiesLanguages(): Promise; - // (undocumented) - generateEntityLanguages(entityRef: string, url: string): Promise; - // (undocumented) - getEntitiesOverview(): Promise; - // (undocumented) getEntityLanguages(entityRef: string): Promise; // (undocumented) - getLinguistResults(dir: string): Promise; - // (undocumented) processEntities(): Promise; } diff --git a/plugins/linguist-backend/src/api/LinguistBackendApi.test.ts b/plugins/linguist-backend/src/api/LinguistBackendClient.test.ts similarity index 97% rename from plugins/linguist-backend/src/api/LinguistBackendApi.test.ts rename to plugins/linguist-backend/src/api/LinguistBackendClient.test.ts index b0f20f0207..58aae70106 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendApi.test.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendClient.test.ts @@ -24,7 +24,7 @@ import { CatalogApi, GetEntitiesResponse } from '@backstage/catalog-client'; import { Results } from 'linguist-js/dist/types'; import { DateTime } from 'luxon'; import { LinguistBackendStore } from '../db'; -import { kindOrDefault, LinguistBackendApi } from './LinguistBackendApi'; +import { kindOrDefault, LinguistBackendClient } from './LinguistBackendClient'; import fs from 'fs-extra'; import { LINGUIST_ANNOTATION } from '@backstage/plugin-linguist-common'; @@ -94,7 +94,7 @@ describe('Linguist backend API', () => { const tokenManager = ServerTokenManager.noop(); - const api = new LinguistBackendApi( + const api = new LinguistBackendClient( logger, store, urlReader, @@ -203,7 +203,7 @@ describe('Linguist backend API', () => { }); it('should get entity overview with stale items', async () => { - const apiWithAge = new LinguistBackendApi( + const apiWithAge = new LinguistBackendClient( logger, store, urlReader, @@ -326,7 +326,7 @@ describe('Linguist backend API', () => { }); it('should generate languages for entities using defined batch size', async () => { - const apiWithBatchSize = new LinguistBackendApi( + const apiWithBatchSize = new LinguistBackendClient( logger, store, urlReader, diff --git a/plugins/linguist-backend/src/api/LinguistBackendApi.ts b/plugins/linguist-backend/src/api/LinguistBackendClient.ts similarity index 90% rename from plugins/linguist-backend/src/api/LinguistBackendApi.ts rename to plugins/linguist-backend/src/api/LinguistBackendClient.ts index f00aebcee1..890e6438b2 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendApi.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendClient.ts @@ -39,9 +39,16 @@ import { } from '@backstage/catalog-model'; import { assertError } from '@backstage/errors'; import { HumanDuration } from '@backstage/types'; +import { Results } from 'linguist-js/dist/types'; /** @public */ -export class LinguistBackendApi { +export interface LinguistBackendApi { + getEntityLanguages(entityRef: string): Promise; + processEntities(): Promise; +} + +/** @public */ +export class LinguistBackendClient implements LinguistBackendApi { private readonly logger: Logger; private readonly store: LinguistBackendStore; private readonly urlReader: UrlReader; @@ -77,13 +84,13 @@ export class LinguistBackendApi { this.linguistJsOptions = linguistJsOptions; } - public async getEntityLanguages(entityRef: string): Promise { + async getEntityLanguages(entityRef: string): Promise { this.logger?.debug(`Getting languages for entity "${entityRef}"`); return this.store.getEntityResults(entityRef); } - public async processEntities() { + async processEntities(): Promise { this.logger?.info('Updating list of entities'); await this.addNewEntities(); @@ -93,7 +100,8 @@ export class LinguistBackendApi { await this.generateEntitiesLanguages(); } - public async addNewEntities() { + /** @internal */ + async addNewEntities(): Promise { const annotationKey = this.useSourceLocation ? ANNOTATION_SOURCE_LOCATION : LINGUIST_ANNOTATION; @@ -115,7 +123,8 @@ export class LinguistBackendApi { }); } - public async generateEntitiesLanguages() { + /** @internal */ + async generateEntitiesLanguages(): Promise { const entitiesOverview = await this.getEntitiesOverview(); this.logger?.info( `Entities overview: Entity: ${entitiesOverview.entityCount}, Processed: ${entitiesOverview.processedCount}, Pending: ${entitiesOverview.pendingCount}, Stale ${entitiesOverview.staleCount}`, @@ -152,7 +161,8 @@ export class LinguistBackendApi { } } - public async getEntitiesOverview(): Promise { + /** @internal */ + async getEntitiesOverview(): Promise { this.logger?.debug('Getting pending entities'); const processedEntities = await this.store.getProcessedEntities(); @@ -178,7 +188,8 @@ export class LinguistBackendApi { return entitiesOverview; } - public async generateEntityLanguages( + /** @internal */ + async generateEntityLanguages( entityRef: string, url: string, ): Promise { @@ -230,13 +241,14 @@ export class LinguistBackendApi { } } - public async getLinguistResults(dir: string) { + /** @internal */ + async getLinguistResults(dir: string): Promise { const results = await linguist(dir, this.linguistJsOptions); return results; } } -export function kindOrDefault(kind?: string[]) { +export function kindOrDefault(kind?: string[]): string[] { if (!kind || kind.length === 0) { return ['API', 'Component', 'Template']; } diff --git a/plugins/linguist-backend/src/api/index.ts b/plugins/linguist-backend/src/api/index.ts index a88fa051d4..53b427b3a7 100644 --- a/plugins/linguist-backend/src/api/index.ts +++ b/plugins/linguist-backend/src/api/index.ts @@ -14,4 +14,5 @@ * limitations under the License. */ -export { LinguistBackendApi } from './LinguistBackendApi'; +export { LinguistBackendClient } from './LinguistBackendClient'; +export type { LinguistBackendApi } from './LinguistBackendClient'; diff --git a/plugins/linguist-backend/src/index.ts b/plugins/linguist-backend/src/index.ts index da1e0ffb00..8969880618 100644 --- a/plugins/linguist-backend/src/index.ts +++ b/plugins/linguist-backend/src/index.ts @@ -15,6 +15,7 @@ */ export * from './service/router'; -export { LinguistBackendApi } from './api'; +export { LinguistBackendClient } from './api'; +export type { LinguistBackendApi } from './api'; export { LinguistBackendDatabase } from './db'; export type { LinguistBackendStore } from './db'; diff --git a/plugins/linguist-backend/src/service/router.test.ts b/plugins/linguist-backend/src/service/router.test.ts index b8bde9d35d..7ab8f1c238 100644 --- a/plugins/linguist-backend/src/service/router.test.ts +++ b/plugins/linguist-backend/src/service/router.test.ts @@ -73,7 +73,7 @@ describe('createRouter', () => { const router = await createRouter( { schedule: schedule, age: { days: 30 }, useSourceLocation: false }, { - linguistBackendApi, + linguistBackendApi: linguistBackendApi, discovery: testDiscovery, database: createDatabase(), reader: mockUrlReader, diff --git a/plugins/linguist-backend/src/service/router.ts b/plugins/linguist-backend/src/service/router.ts index 4c636aa854..21f03e08a3 100644 --- a/plugins/linguist-backend/src/service/router.ts +++ b/plugins/linguist-backend/src/service/router.ts @@ -24,7 +24,7 @@ import { import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; -import { LinguistBackendApi } from '../api'; +import { LinguistBackendApi, LinguistBackendClient } from '../api'; import { LinguistBackendDatabase } from '../db'; import { PluginTaskScheduler, @@ -77,9 +77,9 @@ export async function createRouter( const catalogClient = new CatalogClient({ discoveryApi: discovery }); - const linguistBackendApi = + const linguistBackendClient = routerOptions.linguistBackendApi || - new LinguistBackendApi( + new LinguistBackendClient( logger, linguistBackendStore, reader, @@ -103,7 +103,7 @@ export async function createRouter( initialDelay: schedule.initialDelay, scope: schedule.scope, fn: async () => { - await linguistBackendApi.processEntities(); + await linguistBackendClient.processEntities(); }, }); } @@ -125,7 +125,7 @@ export async function createRouter( throw new Error('No entityRef was provided'); } - const entityLanguages = await linguistBackendApi.getEntityLanguages( + const entityLanguages = await linguistBackendClient.getEntityLanguages( entityRef as string, ); res.status(200).json(entityLanguages); From 763bf4c8f29de23d9ccef692981f7ec1ffb60d3e Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Fri, 7 Apr 2023 08:07:51 -0500 Subject: [PATCH 11/29] Refactored README Signed-off-by: Andre Wanlin --- plugins/linguist-backend/README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/plugins/linguist-backend/README.md b/plugins/linguist-backend/README.md index 46c067fb5a..c1201fc70a 100644 --- a/plugins/linguist-backend/README.md +++ b/plugins/linguist-backend/README.md @@ -57,7 +57,11 @@ Here's how to get the backend up and running: 4. Now run `yarn start-backend` from the repo root 5. Finally open `http://localhost:7007/api/linguist/health` in a browser and it should return `{"status":"ok"}` -## Batch Size +## Plugin Option + +The Linguist backend has various plugin options that you can provide to the `createRouter` function in your `packages/backend/src/plugins/linguist.ts` file that will allow you to configure various aspects of how it works. The following sections go into the details of these options + +### Batch Size The Linguist backend is setup to process entities by acting as a queue where it will pull down all the applicable entities from the Catalog and add them to it's database (saving just the `entityRef`). Then it will grab the `n` oldest entities that have not been processed to determine their languages and process them. To control the batch size simply provide that to the `createRouter` function in your `packages/backend/src/plugins/linguist.ts` like this: @@ -67,7 +71,7 @@ return createRouter({ schedule: schedule, batchSize: 40 }, { ...env }); **Note:** The default batch size is 20 -## Kind +### Kind The default setup only processes entities of kind `['API', 'Component', 'Template']`. To control the `kind` that are processed provide that to the `createRouter` function in your `packages/backend/src/plugins/linguist.ts` like this: @@ -75,7 +79,7 @@ The default setup only processes entities of kind `['API', 'Component', 'Templat return createRouter({ schedule: schedule, kind: ['Component'] }, { ...env }); ``` -## Refresh +### Refresh The default setup will only generate the language breakdown for entities with the linguist annotation that have not been generated yet. If you want this process to also refresh the data you can do so by adding the `age` (as a `HumanDuration`) in your `packages/backend/src/plugins/linguist.ts` when you call `createRouter`: @@ -85,7 +89,7 @@ return createRouter({ schedule: schedule, age: { days: 30 } }, { ...env }); With the `age` setup like this if the language breakdown is older than 15 days it will get regenerated. It's recommended that if you choose to use this configuration to set it to a large value - 30, 90, or 180 - as this data generally does not change drastically. -## Linguist JS options +### Linguist JS options The default setup will use the default [linguist-js](https://www.npmjs.com/package/linguist-js) options, a full list of the available options can be found [here](https://www.npmjs.com/package/linguist-js#API). @@ -96,7 +100,7 @@ return createRouter( ); ``` -## Use Source Location +### Use Source Location You may wish to use the `backstage.io/source-location` annotation over using the `backstage.io/linguist` as you may not be able to quickly add that annotation to your Entities. To do this you'll just need to set the `useSourceLocation` boolean to `true` in your `packages/backend/src/plugins/linguist.ts` when you call `createRouter`: From 9febc291d62339a5e61827f5a2a591f1baa4a175 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Fri, 7 Apr 2023 11:14:04 -0500 Subject: [PATCH 12/29] Unprocessed should be processed before stale Signed-off-by: Andre Wanlin --- plugins/linguist-backend/api-report.md | 8 ++++---- .../src/api/LinguistBackendClient.test.ts | 2 +- .../src/api/LinguistBackendClient.ts | 2 +- .../src/db/LinguistBackendDatabase.test.ts | 14 ++++++++++++++ .../src/db/LinguistBackendDatabase.ts | 8 ++++---- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/plugins/linguist-backend/api-report.md b/plugins/linguist-backend/api-report.md index 795271f46c..8b17b997dd 100644 --- a/plugins/linguist-backend/api-report.md +++ b/plugins/linguist-backend/api-report.md @@ -60,9 +60,9 @@ export class LinguistBackendDatabase implements LinguistBackendStore { // (undocumented) getEntityResults(entityRef: string): Promise; // (undocumented) - getProcessedEntities(): Promise; + getProcessedEntities(): Promise; // (undocumented) - getUnprocessedEntities(): Promise; + getUnprocessedEntities(): Promise; // (undocumented) insertEntityResults(entityLanguages: EntityResults): Promise; // (undocumented) @@ -74,9 +74,9 @@ export interface LinguistBackendStore { // (undocumented) getEntityResults(entityRef: string): Promise; // (undocumented) - getProcessedEntities(): Promise; + getProcessedEntities(): Promise; // (undocumented) - getUnprocessedEntities(): Promise; + getUnprocessedEntities(): Promise; // (undocumented) insertEntityResults(entityLanguages: EntityResults): Promise; // (undocumented) diff --git a/plugins/linguist-backend/src/api/LinguistBackendClient.test.ts b/plugins/linguist-backend/src/api/LinguistBackendClient.test.ts index 58aae70106..35336bb5e9 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendClient.test.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendClient.test.ts @@ -234,10 +234,10 @@ describe('Linguist backend API', () => { expect(overview.staleCount).toEqual(1); expect(overview.pendingCount).toEqual(4); expect(overview.filteredEntities).toEqual([ - 'component:default/stale-service-two', 'component:default/service-three', 'component:default/service-four', 'component:default/service-five', + 'component:default/stale-service-two', ]); }); diff --git a/plugins/linguist-backend/src/api/LinguistBackendClient.ts b/plugins/linguist-backend/src/api/LinguistBackendClient.ts index 890e6438b2..a2f6e1ef67 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendClient.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendClient.ts @@ -175,7 +175,7 @@ export class LinguistBackendClient implements LinguistBackendApi { .map(pe => pe.entityRef); const unprocessedEntities = await this.store.getUnprocessedEntities(); - const filteredEntities = staleEntities.concat(unprocessedEntities); + const filteredEntities = unprocessedEntities.concat(staleEntities); const entitiesOverview: EntitiesOverview = { entityCount: unprocessedEntities.length + processedEntities.length, diff --git a/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts b/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts index f4e0bba627..1ab092b953 100644 --- a/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts +++ b/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts @@ -134,6 +134,13 @@ describe('Linguist database', () => { expect(unprocessedEntities).toMatchObject(validUnprocessedEntities); }); + it('should return string[] when there is no unprocessed entities', async () => { + await testDbClient('entity_result').delete(); + const unprocessedEntities = await store.getUnprocessedEntities(); + + expect(unprocessedEntities).toMatchObject([]); + }); + it('should be able to return processed entities', async () => { const validProcessedEntities: ProcessedEntity[] = [ { @@ -155,6 +162,13 @@ describe('Linguist database', () => { expect(processedEntities).toMatchObject(validProcessedEntities); }); + it('should return string[] when there is no processed entities', async () => { + await testDbClient('entity_result').delete(); + const unprocessedEntities = await store.getProcessedEntities(); + + expect(unprocessedEntities).toMatchObject([]); + }); + it('should insert new entities and ignore duplicates', async () => { const before = testDbClient.count('entity_result'); diff --git a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts index c045d0a4a9..5a165695da 100644 --- a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts +++ b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts @@ -35,8 +35,8 @@ export interface LinguistBackendStore { insertEntityResults(entityLanguages: EntityResults): Promise; insertNewEntity(entityRef: string): Promise; getEntityResults(entityRef: string): Promise; - getProcessedEntities(): Promise; - getUnprocessedEntities(): Promise; + getProcessedEntities(): Promise; + getUnprocessedEntities(): Promise; } const migrationsDir = resolvePackagePath( @@ -108,7 +108,7 @@ export class LinguistBackendDatabase implements LinguistBackendStore { } } - async getProcessedEntities(): Promise { + async getProcessedEntities(): Promise { const rawEntities = await this.db('entity_result') .whereNotNull('processed_date') .whereNotNull('languages'); @@ -140,7 +140,7 @@ export class LinguistBackendDatabase implements LinguistBackendStore { return processedEntities; } - async getUnprocessedEntities(): Promise { + async getUnprocessedEntities(): Promise { const rawEntities = await this.db('entity_result') // TODO(ahhhndre) processed_date should always be null as well but it had a default to the current date // once the default has been removed and released, we can then come back an enable this check From 25659d38d6486019a167eb4c57f454c03d711e3d Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Fri, 7 Apr 2023 11:27:20 -0500 Subject: [PATCH 13/29] Clean up orphan entities Signed-off-by: Andre Wanlin --- plugins/linguist-backend/api-report.md | 8 +++++++ .../src/api/LinguistBackendClient.test.ts | 22 ++++++++++++++++++- .../src/api/LinguistBackendClient.ts | 22 +++++++++++++++++-- .../src/db/LinguistBackendDatabase.test.ts | 21 ++++++++++++++++-- .../src/db/LinguistBackendDatabase.ts | 22 +++++++++++++++++++ 5 files changed, 90 insertions(+), 5 deletions(-) diff --git a/plugins/linguist-backend/api-report.md b/plugins/linguist-backend/api-report.md index 8b17b997dd..1b2c4af832 100644 --- a/plugins/linguist-backend/api-report.md +++ b/plugins/linguist-backend/api-report.md @@ -58,6 +58,10 @@ export class LinguistBackendDatabase implements LinguistBackendStore { // (undocumented) static create(knex: Knex): Promise; // (undocumented) + deleteEntity(entityRef: string): Promise; + // (undocumented) + getAllEntities(): Promise; + // (undocumented) getEntityResults(entityRef: string): Promise; // (undocumented) getProcessedEntities(): Promise; @@ -71,6 +75,10 @@ export class LinguistBackendDatabase implements LinguistBackendStore { // @public (undocumented) export interface LinguistBackendStore { + // (undocumented) + deleteEntity(entityRef: string): Promise; + // (undocumented) + getAllEntities(): Promise; // (undocumented) getEntityResults(entityRef: string): Promise; // (undocumented) diff --git a/plugins/linguist-backend/src/api/LinguistBackendClient.test.ts b/plugins/linguist-backend/src/api/LinguistBackendClient.test.ts index 35336bb5e9..1b0a4f4673 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendClient.test.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendClient.test.ts @@ -79,6 +79,8 @@ describe('Linguist backend API', () => { getEntityResults: jest.fn(), getProcessedEntities: jest.fn(), getUnprocessedEntities: jest.fn(), + getAllEntities: jest.fn(), + deleteEntity: jest.fn(), }; const urlReader: jest.Mocked = { @@ -140,7 +142,7 @@ describe('Linguist backend API', () => { }); }); - it('should add new entities', async () => { + it('should insert new entities', async () => { const testEntityListResponse: GetEntitiesResponse = { items: [ { @@ -172,6 +174,24 @@ describe('Linguist backend API', () => { expect(store.insertNewEntity).toHaveBeenCalledTimes(3); }); + it('should delete entities not in Catalog', async () => { + store.getAllEntities.mockResolvedValue([ + 'component:default/service-one', + 'component:default/stale-service-two', + ]); + + catalogApi.getEntityByRef.mockResolvedValueOnce({ + apiVersion: 'backstage.io/v1beta1', + metadata: { + name: 'service-one', + }, + kind: 'Component', + }); + + await api.cleanEntities(); + expect(store.deleteEntity).toHaveBeenCalledTimes(1); + }); + it('should get default entity overview', async () => { store.getProcessedEntities.mockResolvedValue([ { diff --git a/plugins/linguist-backend/src/api/LinguistBackendClient.ts b/plugins/linguist-backend/src/api/LinguistBackendClient.ts index a2f6e1ef67..0bd6370ac3 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendClient.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendClient.ts @@ -92,11 +92,12 @@ export class LinguistBackendClient implements LinguistBackendApi { async processEntities(): Promise { this.logger?.info('Updating list of entities'); - await this.addNewEntities(); - this.logger?.info('Processing applicable entities through Linguist'); + this.logger?.info('Cleaning list of entities'); + await this.cleanEntities(); + this.logger?.info('Processing applicable entities through Linguist'); await this.generateEntitiesLanguages(); } @@ -123,6 +124,23 @@ export class LinguistBackendClient implements LinguistBackendApi { }); } + /** @internal */ + async cleanEntities(): Promise { + this.logger?.info('Cleaning entities in Linguist queue'); + const allEntities = await this.store.getAllEntities(); + + for (const entityRef of allEntities) { + const result = await this.catalogApi.getEntityByRef(entityRef); + + if (!result) { + this.logger?.info( + `Entity ${entityRef} was not found in the Catalog, it will be deleted`, + ); + await this.store.deleteEntity(entityRef); + } + } + } + /** @internal */ async generateEntitiesLanguages(): Promise { const entitiesOverview = await this.getEntitiesOverview(); diff --git a/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts b/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts index 1ab092b953..5998ddfc1e 100644 --- a/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts +++ b/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts @@ -170,14 +170,31 @@ describe('Linguist database', () => { }); it('should insert new entities and ignore duplicates', async () => { - const before = testDbClient.count('entity_result'); + const before = testDbClient.from('entity_result').count(); await store.insertNewEntity('component:/default/new-entity-one'); await store.insertNewEntity('component:/default/new-entity-two'); await store.insertNewEntity('template:default/pull-request'); - const after = testDbClient.count('entity_result'); + const after = testDbClient.from('entity_result').count(); expect(before).toEqual(after); }); + + it('should get all entities', async () => { + const allEntities = await store.getAllEntities(); + + expect(allEntities.length).toEqual(5); + }); + + it('should delete entity by its entityRef', async () => { + const before = await testDbClient.from('entity_result').count(); + + await store.deleteEntity('template:default/create-react-app-template'); + + const after = await testDbClient.from('entity_result').count(); + + expect(before).toEqual([{ 'count(*)': 5 }]); + expect(after).toEqual([{ 'count(*)': 4 }]); + }); }); diff --git a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts index 5a165695da..2a15b2d823 100644 --- a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts +++ b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts @@ -37,6 +37,8 @@ export interface LinguistBackendStore { getEntityResults(entityRef: string): Promise; getProcessedEntities(): Promise; getUnprocessedEntities(): Promise; + getAllEntities(): Promise; + deleteEntity(entityRef: string): Promise; } const migrationsDir = resolvePackagePath( @@ -158,4 +160,24 @@ export class LinguistBackendDatabase implements LinguistBackendStore { return unprocessedEntities; } + + async getAllEntities(): Promise { + const rawEntities = await this.db('entity_result'); + + if (!rawEntities) { + return []; + } + + const allEntities = rawEntities.map(rawEntity => { + return rawEntity.entity_ref; + }); + + return allEntities; + } + + async deleteEntity(entityRef: string): Promise { + await this.db('entity_result') + .where('entity_ref', entityRef) + .delete(); + } } From 75fe31e5a4f3e05130e7c14f72640d1ac4467bb1 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Fri, 7 Apr 2023 11:38:52 -0500 Subject: [PATCH 14/29] Updated changeset to match changes Signed-off-by: Andre Wanlin --- .changeset/shaggy-gorillas-occur.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.changeset/shaggy-gorillas-occur.md b/.changeset/shaggy-gorillas-occur.md index c122032e7f..94b0f6c21d 100644 --- a/.changeset/shaggy-gorillas-occur.md +++ b/.changeset/shaggy-gorillas-occur.md @@ -2,4 +2,13 @@ '@backstage/plugin-linguist-backend': patch --- -Added tests for the `LinguistBackendDatabase` and `LinguistBackendApi` which included refactoring to better support using SQLite and removed the default from the `processes_date` column +Several improvements to the Linguist backend have been made: + +- Added tests for the `LinguistBackendDatabase` and `LinguistBackendApi` +- Added support for using SQLite as a database, helpful for local development +- Removed the default from the `processes_date` column +- Converted the `LinguistBackendApi` into an Interface +- Added the `LinguistBackendClient` which implements the `LinguistBackendApi` Interface +- Unprocessed entities will get processed before stale entities +- Entities in the Linguist database but not in the Catalog anymore will be deleted +- Improved the README's headings From 62631dbbd7182b72e8bffae2b0fcd142bf4b9188 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Fri, 14 Apr 2023 20:14:52 -0500 Subject: [PATCH 15/29] Removed export Signed-off-by: Andre Wanlin --- plugins/linguist-backend/api-report.md | 21 ------------------- plugins/linguist-backend/src/api/index.ts | 1 - plugins/linguist-backend/src/index.ts | 1 - .../linguist-backend/src/service/router.ts | 3 ++- 4 files changed, 2 insertions(+), 24 deletions(-) diff --git a/plugins/linguist-backend/api-report.md b/plugins/linguist-backend/api-report.md index 1b2c4af832..0c70f936c6 100644 --- a/plugins/linguist-backend/api-report.md +++ b/plugins/linguist-backend/api-report.md @@ -3,7 +3,6 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { CatalogApi } from '@backstage/catalog-client'; import { EntityResults } from '@backstage/plugin-linguist-common'; import express from 'express'; import { HumanDuration } from '@backstage/types'; @@ -32,26 +31,6 @@ export interface LinguistBackendApi { processEntities(): Promise; } -// @public (undocumented) -export class LinguistBackendClient implements LinguistBackendApi { - constructor( - logger: Logger, - store: LinguistBackendStore, - urlReader: UrlReader, - tokenManager: TokenManager, - catalogApi: CatalogApi, - age?: HumanDuration, - batchSize?: number, - useSourceLocation?: boolean, - kind?: string[], - linguistJsOptions?: Record, - ); - // (undocumented) - getEntityLanguages(entityRef: string): Promise; - // (undocumented) - processEntities(): Promise; -} - // @public (undocumented) export class LinguistBackendDatabase implements LinguistBackendStore { constructor(db: Knex); diff --git a/plugins/linguist-backend/src/api/index.ts b/plugins/linguist-backend/src/api/index.ts index 53b427b3a7..b3f73587e7 100644 --- a/plugins/linguist-backend/src/api/index.ts +++ b/plugins/linguist-backend/src/api/index.ts @@ -14,5 +14,4 @@ * limitations under the License. */ -export { LinguistBackendClient } from './LinguistBackendClient'; export type { LinguistBackendApi } from './LinguistBackendClient'; diff --git a/plugins/linguist-backend/src/index.ts b/plugins/linguist-backend/src/index.ts index 8969880618..6d3c86a8f2 100644 --- a/plugins/linguist-backend/src/index.ts +++ b/plugins/linguist-backend/src/index.ts @@ -15,7 +15,6 @@ */ export * from './service/router'; -export { LinguistBackendClient } from './api'; export type { LinguistBackendApi } from './api'; export { LinguistBackendDatabase } from './db'; export type { LinguistBackendStore } from './db'; diff --git a/plugins/linguist-backend/src/service/router.ts b/plugins/linguist-backend/src/service/router.ts index 21f03e08a3..9e45438ec8 100644 --- a/plugins/linguist-backend/src/service/router.ts +++ b/plugins/linguist-backend/src/service/router.ts @@ -24,7 +24,7 @@ import { import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; -import { LinguistBackendApi, LinguistBackendClient } from '../api'; +import { LinguistBackendApi } from '../api'; import { LinguistBackendDatabase } from '../db'; import { PluginTaskScheduler, @@ -32,6 +32,7 @@ import { } from '@backstage/backend-tasks'; import { HumanDuration } from '@backstage/types'; import { CatalogClient } from '@backstage/catalog-client'; +import { LinguistBackendClient } from '../api/LinguistBackendClient'; /** @public */ export interface PluginOptions { From 0f439d6508789da9d5e301d46f62d240e9f10d41 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Sat, 22 Apr 2023 10:12:28 -0500 Subject: [PATCH 16/29] Removed database export Signed-off-by: Andre Wanlin --- plugins/linguist-backend/api-report.md | 42 -------------------------- plugins/linguist-backend/src/index.ts | 2 -- 2 files changed, 44 deletions(-) diff --git a/plugins/linguist-backend/api-report.md b/plugins/linguist-backend/api-report.md index 0c70f936c6..294d569ff7 100644 --- a/plugins/linguist-backend/api-report.md +++ b/plugins/linguist-backend/api-report.md @@ -3,16 +3,13 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { EntityResults } from '@backstage/plugin-linguist-common'; import express from 'express'; import { HumanDuration } from '@backstage/types'; -import { Knex } from 'knex'; import { Languages } from '@backstage/plugin-linguist-common'; import { Logger } from 'winston'; import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; -import { ProcessedEntity } from '@backstage/plugin-linguist-common'; import { TaskScheduleDefinition } from '@backstage/backend-tasks'; import { TokenManager } from '@backstage/backend-common'; import { UrlReader } from '@backstage/backend-common'; @@ -31,45 +28,6 @@ export interface LinguistBackendApi { processEntities(): Promise; } -// @public (undocumented) -export class LinguistBackendDatabase implements LinguistBackendStore { - constructor(db: Knex); - // (undocumented) - static create(knex: Knex): Promise; - // (undocumented) - deleteEntity(entityRef: string): Promise; - // (undocumented) - getAllEntities(): Promise; - // (undocumented) - getEntityResults(entityRef: string): Promise; - // (undocumented) - getProcessedEntities(): Promise; - // (undocumented) - getUnprocessedEntities(): Promise; - // (undocumented) - insertEntityResults(entityLanguages: EntityResults): Promise; - // (undocumented) - insertNewEntity(entityRef: string): Promise; -} - -// @public (undocumented) -export interface LinguistBackendStore { - // (undocumented) - deleteEntity(entityRef: string): Promise; - // (undocumented) - getAllEntities(): Promise; - // (undocumented) - getEntityResults(entityRef: string): Promise; - // (undocumented) - getProcessedEntities(): Promise; - // (undocumented) - getUnprocessedEntities(): Promise; - // (undocumented) - insertEntityResults(entityLanguages: EntityResults): Promise; - // (undocumented) - insertNewEntity(entityRef: string): Promise; -} - // @public (undocumented) export interface PluginOptions { // (undocumented) diff --git a/plugins/linguist-backend/src/index.ts b/plugins/linguist-backend/src/index.ts index 6d3c86a8f2..37ecf2d47f 100644 --- a/plugins/linguist-backend/src/index.ts +++ b/plugins/linguist-backend/src/index.ts @@ -16,5 +16,3 @@ export * from './service/router'; export type { LinguistBackendApi } from './api'; -export { LinguistBackendDatabase } from './db'; -export type { LinguistBackendStore } from './db'; From b66092c1349c386b8efc8546e6faed8aa29b7e0d Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Tue, 2 May 2023 15:35:25 -0500 Subject: [PATCH 17/29] Latest changes based on feedback Signed-off-by: Andre Wanlin --- .changeset/shaggy-gorillas-occur.md | 2 ++ plugins/linguist-backend/src/api/LinguistBackendClient.ts | 1 - plugins/linguist-backend/src/db/LinguistBackendDatabase.ts | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.changeset/shaggy-gorillas-occur.md b/.changeset/shaggy-gorillas-occur.md index 94b0f6c21d..149ff6f424 100644 --- a/.changeset/shaggy-gorillas-occur.md +++ b/.changeset/shaggy-gorillas-occur.md @@ -2,6 +2,8 @@ '@backstage/plugin-linguist-backend': patch --- +**BREAKING**: Removed public constructor from `LinguistBackendApi`. Removed export of `LinguistBackendDatabase` and `LinguistBackendDatabase` + Several improvements to the Linguist backend have been made: - Added tests for the `LinguistBackendDatabase` and `LinguistBackendApi` diff --git a/plugins/linguist-backend/src/api/LinguistBackendClient.ts b/plugins/linguist-backend/src/api/LinguistBackendClient.ts index 0bd6370ac3..5a84201bcb 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendClient.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendClient.ts @@ -170,7 +170,6 @@ export class LinguistBackendClient implements LinguistBackendApi { try { await this.generateEntityLanguages(entityRef, url); } catch (error) { - console.log(error); assertError(error); this.logger.error( `Unable to process "${entityRef}" using "${url}", message: ${error.message}, stack: ${error.stack}`, diff --git a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts index 2a15b2d823..12f44a5dca 100644 --- a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts +++ b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts @@ -93,7 +93,7 @@ export class LinguistBackendDatabase implements LinguistBackendStore { .where({ entity_ref: entityRef }) .first(); - if (!entityResults) { + if (!entityResults || !entityResults.languages) { const emptyResults: Languages = { languageCount: 0, totalBytes: 0, @@ -104,7 +104,7 @@ export class LinguistBackendDatabase implements LinguistBackendStore { } try { - return JSON.parse(entityResults.languages as string); + return JSON.parse(entityResults.languages); } catch (error) { throw new Error(`Failed to parse languages for '${entityRef}', ${error}`); } From 7a0010adee499756177e0f2e1c82039f55feb9c6 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Tue, 2 May 2023 15:36:47 -0500 Subject: [PATCH 18/29] Fixed changeset typo Signed-off-by: Andre Wanlin --- .changeset/shaggy-gorillas-occur.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/shaggy-gorillas-occur.md b/.changeset/shaggy-gorillas-occur.md index 149ff6f424..d36b8aef01 100644 --- a/.changeset/shaggy-gorillas-occur.md +++ b/.changeset/shaggy-gorillas-occur.md @@ -2,7 +2,7 @@ '@backstage/plugin-linguist-backend': patch --- -**BREAKING**: Removed public constructor from `LinguistBackendApi`. Removed export of `LinguistBackendDatabase` and `LinguistBackendDatabase` +**BREAKING**: Removed public constructor from `LinguistBackendApi`. Removed export of `LinguistBackendDatabase` and `LinguistBackendStore` Several improvements to the Linguist backend have been made: From 8676ebfd42d8c270fb4b400a6108b3e42c2f9442 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Tue, 2 May 2023 15:38:31 -0500 Subject: [PATCH 19/29] Syntax improvement Signed-off-by: Andre Wanlin --- plugins/linguist-backend/src/db/LinguistBackendDatabase.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts index 12f44a5dca..5a59d18e90 100644 --- a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts +++ b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts @@ -133,7 +133,7 @@ export class LinguistBackendDatabase implements LinguistBackendStore { const processEntity = { entityRef: rawEntity.entity_ref, - processedDate: processedDate, + processedDate, }; return processEntity; From d787cc504a4bc92669c413266fbe46dff5f1d282 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Tue, 2 May 2023 15:40:14 -0500 Subject: [PATCH 20/29] Changed to minor Signed-off-by: Andre Wanlin --- .changeset/shaggy-gorillas-occur.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/shaggy-gorillas-occur.md b/.changeset/shaggy-gorillas-occur.md index d36b8aef01..ce7c2aba3b 100644 --- a/.changeset/shaggy-gorillas-occur.md +++ b/.changeset/shaggy-gorillas-occur.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-linguist-backend': patch +'@backstage/plugin-linguist-backend': minor --- **BREAKING**: Removed public constructor from `LinguistBackendApi`. Removed export of `LinguistBackendDatabase` and `LinguistBackendStore` From cda753a797b5d78fc7db7c2f348c304b6fd241e4 Mon Sep 17 00:00:00 2001 From: Sander Aernouts Date: Thu, 11 May 2023 14:44:10 +0200 Subject: [PATCH 21/29] fix(scaffolder): forward authorization header on event source Signed-off-by: Sander Aernouts --- .changeset/curvy-years-smoke.md | 5 +++++ plugins/scaffolder/package.json | 2 +- plugins/scaffolder/src/api.test.ts | 15 ++++++++++++--- plugins/scaffolder/src/api.ts | 15 ++++++++++++--- yarn.lock | 9 ++++++++- 5 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 .changeset/curvy-years-smoke.md diff --git a/.changeset/curvy-years-smoke.md b/.changeset/curvy-years-smoke.md new file mode 100644 index 0000000000..056056fdff --- /dev/null +++ b/.changeset/curvy-years-smoke.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': patch +--- + +Forward `Authorization` header for `EventSource` when credentials are available. diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index 4b5e34e5de..a80ecec049 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -74,6 +74,7 @@ "@types/react": "^16.13.1 || ^17.0.0", "@uiw/react-codemirror": "^4.9.3", "classnames": "^2.2.6", + "event-source-polyfill": "^1.0.31", "git-url-parse": "^13.0.0", "humanize-duration": "^3.25.1", "immer": "^9.0.1", @@ -108,7 +109,6 @@ "@types/json-schema": "^7.0.9", "@types/node": "^16.11.26", "cross-fetch": "^3.1.5", - "event-source-polyfill": "1.0.25", "msw": "^1.0.0" }, "files": [ diff --git a/plugins/scaffolder/src/api.test.ts b/plugins/scaffolder/src/api.test.ts index e21bda8fd7..b9b1881066 100644 --- a/plugins/scaffolder/src/api.test.ts +++ b/plugins/scaffolder/src/api.test.ts @@ -20,11 +20,14 @@ import { MockFetchApi, setupRequestMockHandlers } from '@backstage/test-utils'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import { ScaffolderClient } from './api'; +import { EventSourcePolyfill } from 'event-source-polyfill'; -const MockedEventSource = global.EventSource as jest.MockedClass< - typeof EventSource +const MockedEventSource = EventSourcePolyfill as jest.MockedClass< + typeof EventSourcePolyfill >; +jest.mock('event-source-polyfill'); + const server = setupServer(); describe('api', () => { @@ -102,6 +105,9 @@ describe('api', () => { }, ); + const token = 'fake-token'; + identityApi.getCredentials.mockResolvedValue({ token: token }); + const next = jest.fn(); await new Promise(complete => { @@ -112,7 +118,10 @@ describe('api', () => { expect(MockedEventSource).toHaveBeenCalledWith( 'http://backstage/api/v2/tasks/a-random-task-id/eventstream', - { withCredentials: true }, + { + withCredentials: true, + headers: { Authorization: `Bearer ${token}` }, + }, ); expect(MockedEventSource.prototype.close).toHaveBeenCalled(); diff --git a/plugins/scaffolder/src/api.ts b/plugins/scaffolder/src/api.ts index 7a3f6c14bb..c67a5c944e 100644 --- a/plugins/scaffolder/src/api.ts +++ b/plugins/scaffolder/src/api.ts @@ -41,6 +41,7 @@ import { } from '@backstage/plugin-scaffolder-react'; import queryString from 'qs'; +import { EventSourcePolyfill } from 'event-source-polyfill'; /** * An API to interact with the scaffolder backend. @@ -224,8 +225,11 @@ export class ScaffolderClient implements ScaffolderApi { params.set('after', String(Number(after))); } - this.discoveryApi.getBaseUrl('scaffolder').then( - baseUrl => { + Promise.all([ + this.discoveryApi.getBaseUrl('scaffolder'), + this.identityApi?.getCredentials(), + ]).then( + ([baseUrl, credentials]) => { const url = `${baseUrl}/v2/tasks/${encodeURIComponent( taskId, )}/eventstream`; @@ -240,7 +244,12 @@ export class ScaffolderClient implements ScaffolderApi { } }; - const eventSource = new EventSource(url, { withCredentials: true }); + const eventSource = new EventSourcePolyfill(url, { + withCredentials: true, + headers: credentials?.token + ? { Authorization: `Bearer ${credentials.token}` } + : {}, + }); eventSource.addEventListener('log', processEvent); eventSource.addEventListener('cancelled', processEvent); eventSource.addEventListener('completion', (event: any) => { diff --git a/yarn.lock b/yarn.lock index e38ea3fff3..bcfa6ab89d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8658,7 +8658,7 @@ __metadata: "@uiw/react-codemirror": ^4.9.3 classnames: ^2.2.6 cross-fetch: ^3.1.5 - event-source-polyfill: 1.0.25 + event-source-polyfill: ^1.0.31 git-url-parse: ^13.0.0 humanize-duration: ^3.25.1 immer: ^9.0.1 @@ -23845,6 +23845,13 @@ __metadata: languageName: node linkType: hard +"event-source-polyfill@npm:^1.0.31": + version: 1.0.31 + resolution: "event-source-polyfill@npm:1.0.31" + checksum: 973f226404e2a1b14ed7ef15c718b89e213b41d7cfeeb1c10937fd09229f13904f3d7c3075ab28ccf858c213007559908eecdd577577330352f53a351383dd75 + languageName: node + linkType: hard + "event-stream@npm:=3.3.4": version: 3.3.4 resolution: "event-stream@npm:3.3.4" From 37ad4a0cac50bc32f23cd583991f8d24c7cd25b1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 17:56:02 +0000 Subject: [PATCH 22/29] fix(deps): update dependency qs to v6.11.2 Signed-off-by: Renovate Bot --- yarn.lock | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/yarn.lock b/yarn.lock index f91790c2a5..4756ea6253 100644 --- a/yarn.lock +++ b/yarn.lock @@ -34570,7 +34570,7 @@ __metadata: languageName: node linkType: hard -"qs@npm:6.11.0, qs@npm:^6.10.1, qs@npm:^6.10.2, qs@npm:^6.11.0, qs@npm:^6.9.1, qs@npm:^6.9.4, qs@npm:^6.9.6": +"qs@npm:6.11.0": version: 6.11.0 resolution: "qs@npm:6.11.0" dependencies: @@ -34579,6 +34579,15 @@ __metadata: languageName: node linkType: hard +"qs@npm:^6.10.1, qs@npm:^6.10.2, qs@npm:^6.11.0, qs@npm:^6.9.1, qs@npm:^6.9.4, qs@npm:^6.9.6": + version: 6.11.2 + resolution: "qs@npm:6.11.2" + dependencies: + side-channel: ^1.0.4 + checksum: e812f3c590b2262548647d62f1637b6989cc56656dc960b893fe2098d96e1bd633f36576f4cd7564dfbff9db42e17775884db96d846bebe4f37420d073ecdc0b + languageName: node + linkType: hard + "qs@npm:~6.5.2": version: 6.5.3 resolution: "qs@npm:6.5.3" From 3d3c4fe480e7314154afd9a20ad339b23a430715 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 17:56:36 +0000 Subject: [PATCH 23/29] fix(deps): update docusaurus monorepo to v0.0.0-5580 Signed-off-by: Renovate Bot --- microsite/package.json | 8 +- microsite/yarn.lock | 470 ++++++++++++++++++++++++----------------- 2 files changed, 283 insertions(+), 195 deletions(-) diff --git a/microsite/package.json b/microsite/package.json index 601a234688..98301ce2d0 100644 --- a/microsite/package.json +++ b/microsite/package.json @@ -17,7 +17,7 @@ "docusaurus": "docusaurus" }, "devDependencies": { - "@docusaurus/module-type-aliases": "0.0.0-5569", + "@docusaurus/module-type-aliases": "0.0.0-5580", "@spotify/prettier-config": "^14.0.0", "@tsconfig/docusaurus": "^1.0.6", "@types/webpack-env": "^1.18.0", @@ -28,9 +28,9 @@ }, "prettier": "@spotify/prettier-config", "dependencies": { - "@docusaurus/core": "0.0.0-5569", - "@docusaurus/plugin-client-redirects": "0.0.0-5569", - "@docusaurus/preset-classic": "0.0.0-5569", + "@docusaurus/core": "0.0.0-5580", + "@docusaurus/plugin-client-redirects": "0.0.0-5580", + "@docusaurus/preset-classic": "0.0.0-5580", "@swc/core": "^1.3.46", "clsx": "^1.1.1", "docusaurus-plugin-sass": "^0.2.3", diff --git a/microsite/yarn.lock b/microsite/yarn.lock index c16f65240e..fe1c123575 100644 --- a/microsite/yarn.lock +++ b/microsite/yarn.lock @@ -1620,9 +1620,9 @@ __metadata: languageName: node linkType: hard -"@docusaurus/core@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/core@npm:0.0.0-5569" +"@docusaurus/core@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/core@npm:0.0.0-5580" dependencies: "@babel/core": ^7.20.12 "@babel/generator": ^7.21.1 @@ -1634,13 +1634,13 @@ __metadata: "@babel/runtime": ^7.21.0 "@babel/runtime-corejs3": ^7.21.0 "@babel/traverse": ^7.21.2 - "@docusaurus/cssnano-preset": 0.0.0-5569 - "@docusaurus/logger": 0.0.0-5569 - "@docusaurus/mdx-loader": 0.0.0-5569 + "@docusaurus/cssnano-preset": 0.0.0-5580 + "@docusaurus/logger": 0.0.0-5580 + "@docusaurus/mdx-loader": 0.0.0-5580 "@docusaurus/react-loadable": 5.5.2 - "@docusaurus/utils": 0.0.0-5569 - "@docusaurus/utils-common": 0.0.0-5569 - "@docusaurus/utils-validation": 0.0.0-5569 + "@docusaurus/utils": 0.0.0-5580 + "@docusaurus/utils-common": 0.0.0-5580 + "@docusaurus/utils-validation": 0.0.0-5580 "@slorber/static-site-generator-webpack-plugin": ^4.0.7 "@svgr/webpack": ^6.5.1 autoprefixer: ^10.4.13 @@ -1700,41 +1700,41 @@ __metadata: react-dom: ^16.8.4 || ^17.0.0 bin: docusaurus: bin/docusaurus.mjs - checksum: 573746ba85750381b110a98cfb529d8c961523b735c1945016bf8f79ba2cde38d08ecda88dc1be4852d8c9a49121e23ec964b05354fd6f6d26e6f31e71f175c1 + checksum: 1160559109aa834bcd3dc0210ca824ad6cbdea363951c5cba690c1882c78a7da6e1e52b8a5c35fef3bfc23f9f66e84880c354727b7987d371cac8ec6fb77d1bb languageName: node linkType: hard -"@docusaurus/cssnano-preset@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/cssnano-preset@npm:0.0.0-5569" +"@docusaurus/cssnano-preset@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/cssnano-preset@npm:0.0.0-5580" dependencies: cssnano-preset-advanced: ^5.3.10 postcss: ^8.4.21 postcss-sort-media-queries: ^4.3.0 tslib: ^2.5.0 - checksum: 11408a0f9eb1a4a83b2045a0172ef1fbf0523a99fa9d81a8efe9a3a968b449887616ad8d95b0abb55b2bf82a178193c20c0304942c4f2ba51b693b9209031204 + checksum: 4a46af1e9327d6ebbed3f0f6e60cc1ca59d5c07d39be35c42c13b0a64795b0b1565585adfca063798be79100aca425ee0c12f28a816b56686fb97726bec00f40 languageName: node linkType: hard -"@docusaurus/logger@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/logger@npm:0.0.0-5569" +"@docusaurus/logger@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/logger@npm:0.0.0-5580" dependencies: chalk: ^4.1.2 tslib: ^2.5.0 - checksum: 26be75d7180d9b9d525ce5014e1e24c69fd54d1707cabc5acc76b47e3282cfb185613fb4d27f32b5cced92fe80384c68e2d991cf4cabbecca331f570c3e10d9b + checksum: 5792b6672623b70d38871713c47ba88cc0df6c8d224c28ac58d0512b8bf1b672786cc2fa0b1a79cc045c473e1e14b5a9cce4b01e623dd252893294d6aff9d1c2 languageName: node linkType: hard -"@docusaurus/mdx-loader@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/mdx-loader@npm:0.0.0-5569" +"@docusaurus/mdx-loader@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/mdx-loader@npm:0.0.0-5580" dependencies: "@babel/parser": ^7.21.2 "@babel/traverse": ^7.21.2 - "@docusaurus/logger": 0.0.0-5569 - "@docusaurus/utils": 0.0.0-5569 - "@docusaurus/utils-validation": 0.0.0-5569 + "@docusaurus/logger": 0.0.0-5580 + "@docusaurus/utils": 0.0.0-5580 + "@docusaurus/utils-validation": 0.0.0-5580 "@mdx-js/mdx": ^2.1.5 escape-html: ^1.0.3 estree-util-value-to-estree: ^2.1.0 @@ -1744,6 +1744,7 @@ __metadata: image-size: ^1.0.2 mdast-util-mdx: ^2.0.0 mdast-util-to-string: ^3.0.0 + rehype-raw: ^6.1.1 remark-comment: ^1.0.0 remark-directive: ^2.0.1 remark-emoji: ^2.2.0 @@ -1757,16 +1758,16 @@ __metadata: peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: bcf3b58ea3840d648ddc81c090c9905d559f642d8dc495f3fa86b377ae0bdabb6e1486ae2b258efa459a8bcf1da729a4a3a0c144f0de5dc083155b6d5c83099c + checksum: b32812e0bca91a29761fda742a790f67077bb7e8860e20e83f009e297ca49017192fa14051ab82f1a3e66800de9e5a0b561b32ce2dffb4eecc0aee48116d3490 languageName: node linkType: hard -"@docusaurus/module-type-aliases@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/module-type-aliases@npm:0.0.0-5569" +"@docusaurus/module-type-aliases@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/module-type-aliases@npm:0.0.0-5580" dependencies: "@docusaurus/react-loadable": 5.5.2 - "@docusaurus/types": 0.0.0-5569 + "@docusaurus/types": 0.0.0-5580 "@types/history": ^4.7.11 "@types/react": "*" "@types/react-router-config": "*" @@ -1776,19 +1777,19 @@ __metadata: peerDependencies: react: "*" react-dom: "*" - checksum: 08168317c606e44f8ead775bfc23feaeb3a03178a19a07504130a32cbf189926c08ef40cdc4f00a6df0983383ca0145a42df9112451b15a19067d8b6b10d5f89 + checksum: b5b3544f2f2d395c1d406a1b3f03f719aee3aa3fd7603c54af3b09c4fb8bb153cf2ee6697bc9cfc9fb2550a11a3e474a52c9f2c132924e2be60662ffc6379ace languageName: node linkType: hard -"@docusaurus/plugin-client-redirects@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/plugin-client-redirects@npm:0.0.0-5569" +"@docusaurus/plugin-client-redirects@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/plugin-client-redirects@npm:0.0.0-5580" dependencies: - "@docusaurus/core": 0.0.0-5569 - "@docusaurus/logger": 0.0.0-5569 - "@docusaurus/utils": 0.0.0-5569 - "@docusaurus/utils-common": 0.0.0-5569 - "@docusaurus/utils-validation": 0.0.0-5569 + "@docusaurus/core": 0.0.0-5580 + "@docusaurus/logger": 0.0.0-5580 + "@docusaurus/utils": 0.0.0-5580 + "@docusaurus/utils-common": 0.0.0-5580 + "@docusaurus/utils-validation": 0.0.0-5580 eta: ^2.0.1 fs-extra: ^11.1.0 lodash: ^4.17.21 @@ -1796,21 +1797,21 @@ __metadata: peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: f082956089fc889ea67b89c5c19ad6cd0bc9bb35b663a701f05905786938b1302b8667c2b6d0354bcf9efd35496a5920e1871b488fbc6285d0e36cfe3187e84e + checksum: 7414d2026446862505d0c679ad3757107f28100ea76309d2a024ebe78865f2fcf3e3b85c3117361142e319362b5a5b22e6bce968763fdfd0741b6a60d2eb3f4b languageName: node linkType: hard -"@docusaurus/plugin-content-blog@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/plugin-content-blog@npm:0.0.0-5569" +"@docusaurus/plugin-content-blog@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/plugin-content-blog@npm:0.0.0-5580" dependencies: - "@docusaurus/core": 0.0.0-5569 - "@docusaurus/logger": 0.0.0-5569 - "@docusaurus/mdx-loader": 0.0.0-5569 - "@docusaurus/types": 0.0.0-5569 - "@docusaurus/utils": 0.0.0-5569 - "@docusaurus/utils-common": 0.0.0-5569 - "@docusaurus/utils-validation": 0.0.0-5569 + "@docusaurus/core": 0.0.0-5580 + "@docusaurus/logger": 0.0.0-5580 + "@docusaurus/mdx-loader": 0.0.0-5580 + "@docusaurus/types": 0.0.0-5580 + "@docusaurus/utils": 0.0.0-5580 + "@docusaurus/utils-common": 0.0.0-5580 + "@docusaurus/utils-validation": 0.0.0-5580 cheerio: ^1.0.0-rc.12 feed: ^4.2.2 fs-extra: ^11.1.0 @@ -1823,21 +1824,21 @@ __metadata: peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: b9eef1200e1b9bf1cfa7c4d4cb62b7344306d3a096c0cbfc36afd4b973a940b1bf9d6dc62124a0028bbd466313c7ee219444ebe2856588748d2fc33d1f48ae80 + checksum: 0c5d37fa84298ce12b2a0f5c1dea1ee7d72d4b28c545fd081f3c3e5f93f15a980d877dcf88718aa1f4b8a03abf3d0c8c0df10d0a2ef22aac7a8a96d2786f91c2 languageName: node linkType: hard -"@docusaurus/plugin-content-docs@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/plugin-content-docs@npm:0.0.0-5569" +"@docusaurus/plugin-content-docs@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/plugin-content-docs@npm:0.0.0-5580" dependencies: - "@docusaurus/core": 0.0.0-5569 - "@docusaurus/logger": 0.0.0-5569 - "@docusaurus/mdx-loader": 0.0.0-5569 - "@docusaurus/module-type-aliases": 0.0.0-5569 - "@docusaurus/types": 0.0.0-5569 - "@docusaurus/utils": 0.0.0-5569 - "@docusaurus/utils-validation": 0.0.0-5569 + "@docusaurus/core": 0.0.0-5580 + "@docusaurus/logger": 0.0.0-5580 + "@docusaurus/mdx-loader": 0.0.0-5580 + "@docusaurus/module-type-aliases": 0.0.0-5580 + "@docusaurus/types": 0.0.0-5580 + "@docusaurus/utils": 0.0.0-5580 + "@docusaurus/utils-validation": 0.0.0-5580 "@types/react-router-config": ^5.0.6 combine-promises: ^1.1.0 fs-extra: ^11.1.0 @@ -1850,133 +1851,133 @@ __metadata: peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: 544d51dbb6c9037916aa8e3ed493624029ffdac0d0da70cfdb74f1d431a8da071eeeb0d567be6f63d48f9922f1594f08f950478a4ea427fae8001ff0b4ac5d83 + checksum: fa58801bf0cebf055ed11adbadc3f27f7ed0ddb95174ab60bab76a21f759dc7abdfdd15f32141f7e259090c456634ee2556ac9a7daaaf7f5f6e3666d9f247d00 languageName: node linkType: hard -"@docusaurus/plugin-content-pages@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/plugin-content-pages@npm:0.0.0-5569" +"@docusaurus/plugin-content-pages@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/plugin-content-pages@npm:0.0.0-5580" dependencies: - "@docusaurus/core": 0.0.0-5569 - "@docusaurus/mdx-loader": 0.0.0-5569 - "@docusaurus/types": 0.0.0-5569 - "@docusaurus/utils": 0.0.0-5569 - "@docusaurus/utils-validation": 0.0.0-5569 + "@docusaurus/core": 0.0.0-5580 + "@docusaurus/mdx-loader": 0.0.0-5580 + "@docusaurus/types": 0.0.0-5580 + "@docusaurus/utils": 0.0.0-5580 + "@docusaurus/utils-validation": 0.0.0-5580 fs-extra: ^11.1.0 tslib: ^2.5.0 webpack: ^5.76.0 peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: 16fa680c069d57d9d299ef515487a773c5b89d605cfe5b843392a7bce5e59b539b1855d7e0f2e11878b3021038bff6468b5f4adb9af7dc06884d7207772c2f95 + checksum: 42bf814d9bf5540b8b2f8045f1577c49d3fca4a85069865695ff5dd9fa3f935283f9d46f920c4dd0a28190cf3b342d432b0985636bae9f05280835f73aec8c87 languageName: node linkType: hard -"@docusaurus/plugin-debug@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/plugin-debug@npm:0.0.0-5569" +"@docusaurus/plugin-debug@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/plugin-debug@npm:0.0.0-5580" dependencies: - "@docusaurus/core": 0.0.0-5569 - "@docusaurus/types": 0.0.0-5569 - "@docusaurus/utils": 0.0.0-5569 + "@docusaurus/core": 0.0.0-5580 + "@docusaurus/types": 0.0.0-5580 + "@docusaurus/utils": 0.0.0-5580 fs-extra: ^11.1.0 react-json-view: ^1.21.3 tslib: ^2.5.0 peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: 0e3b8f5a92bc2b2ab193d76be6174b521238da7b4627775a852a4b4ff8690af28ab4586adbb84fc03abc6da276efc7474eed9350577b457e532defeb638d3184 + checksum: dc58659adab3c859daa2e0dfc0b5b588d490a4d2bb313dcfdc9e92d559d0c4145c686b001805ad44a1f2a0f55f87524637a34844a4438cfeec479a2c24ec1c4d languageName: node linkType: hard -"@docusaurus/plugin-google-analytics@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/plugin-google-analytics@npm:0.0.0-5569" +"@docusaurus/plugin-google-analytics@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/plugin-google-analytics@npm:0.0.0-5580" dependencies: - "@docusaurus/core": 0.0.0-5569 - "@docusaurus/types": 0.0.0-5569 - "@docusaurus/utils-validation": 0.0.0-5569 + "@docusaurus/core": 0.0.0-5580 + "@docusaurus/types": 0.0.0-5580 + "@docusaurus/utils-validation": 0.0.0-5580 tslib: ^2.5.0 peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: a3e55d3a616009db499ca61493f9cf308c8ae012ec1ff680bff20c3b0b486f8d5d9552fd5c3d7b198e6c72d8e13ace7808d978bd8d56496953d2fd51d5ce0c32 + checksum: f66a737a2b8b08648f28a279784e527fff34e2dba6f3022ec83b01a7f1781bfa0f208fac1db1fc919c1a01d405755c0816304cf4912e7b434b3924d5a968e6b0 languageName: node linkType: hard -"@docusaurus/plugin-google-gtag@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/plugin-google-gtag@npm:0.0.0-5569" +"@docusaurus/plugin-google-gtag@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/plugin-google-gtag@npm:0.0.0-5580" dependencies: - "@docusaurus/core": 0.0.0-5569 - "@docusaurus/types": 0.0.0-5569 - "@docusaurus/utils-validation": 0.0.0-5569 + "@docusaurus/core": 0.0.0-5580 + "@docusaurus/types": 0.0.0-5580 + "@docusaurus/utils-validation": 0.0.0-5580 "@types/gtag.js": ^0.0.12 tslib: ^2.5.0 peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: d0a7f58bea6d5bfd786120ab6c75ab10a4d67bae1d60aadedba649a516f2f9e49237a96a7764f964c79084ae500b9520761cbf6aab60bd5e43bb976f04dd3194 + checksum: 6b05e7c2a8e4d757660acca1131a0e3abfd21c5544a31e93bd08ff9c75deb5f3b131ed0e47283bbc2f102e00947ab7a64ee4d5a53f300e98fc82ec13c4a6eda1 languageName: node linkType: hard -"@docusaurus/plugin-google-tag-manager@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/plugin-google-tag-manager@npm:0.0.0-5569" +"@docusaurus/plugin-google-tag-manager@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/plugin-google-tag-manager@npm:0.0.0-5580" dependencies: - "@docusaurus/core": 0.0.0-5569 - "@docusaurus/types": 0.0.0-5569 - "@docusaurus/utils-validation": 0.0.0-5569 + "@docusaurus/core": 0.0.0-5580 + "@docusaurus/types": 0.0.0-5580 + "@docusaurus/utils-validation": 0.0.0-5580 tslib: ^2.5.0 peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: acafad7eea5ba06c62addcef4f3cfaa6c43193827ec4b6c2d39ed2799df5301defa8ac8c73d49e35c328bd118c78412745f088764d97eac00da090be8e7906e5 + checksum: 8904b150b2faa05dc49fd957ca8c392605e53073d031dfedb098be2abae924c3faf4f5e26ecfea91fe452695699801de9882a85a4371a1a35ca05fd57de232ae languageName: node linkType: hard -"@docusaurus/plugin-sitemap@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/plugin-sitemap@npm:0.0.0-5569" +"@docusaurus/plugin-sitemap@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/plugin-sitemap@npm:0.0.0-5580" dependencies: - "@docusaurus/core": 0.0.0-5569 - "@docusaurus/logger": 0.0.0-5569 - "@docusaurus/types": 0.0.0-5569 - "@docusaurus/utils": 0.0.0-5569 - "@docusaurus/utils-common": 0.0.0-5569 - "@docusaurus/utils-validation": 0.0.0-5569 + "@docusaurus/core": 0.0.0-5580 + "@docusaurus/logger": 0.0.0-5580 + "@docusaurus/types": 0.0.0-5580 + "@docusaurus/utils": 0.0.0-5580 + "@docusaurus/utils-common": 0.0.0-5580 + "@docusaurus/utils-validation": 0.0.0-5580 fs-extra: ^11.1.0 sitemap: ^7.1.1 tslib: ^2.5.0 peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: 53fd5944a1cfbbe252e3f4398acf8aea3605ca3c5fb9ec48700ce6c162744bae541ccecd1d71e884777dfc2e98c6fa9a01b0f98829e6018e59003c3d4f4a1612 + checksum: 4243cf4ed649d8e9a29b122ce23f1a6194cc5d7fc244e33326320315d8ff2756b18b9d85a0f0d49df33b2e68224c1b747e87ad5fa9944e7572bc761f641c5b77 languageName: node linkType: hard -"@docusaurus/preset-classic@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/preset-classic@npm:0.0.0-5569" +"@docusaurus/preset-classic@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/preset-classic@npm:0.0.0-5580" dependencies: - "@docusaurus/core": 0.0.0-5569 - "@docusaurus/plugin-content-blog": 0.0.0-5569 - "@docusaurus/plugin-content-docs": 0.0.0-5569 - "@docusaurus/plugin-content-pages": 0.0.0-5569 - "@docusaurus/plugin-debug": 0.0.0-5569 - "@docusaurus/plugin-google-analytics": 0.0.0-5569 - "@docusaurus/plugin-google-gtag": 0.0.0-5569 - "@docusaurus/plugin-google-tag-manager": 0.0.0-5569 - "@docusaurus/plugin-sitemap": 0.0.0-5569 - "@docusaurus/theme-classic": 0.0.0-5569 - "@docusaurus/theme-common": 0.0.0-5569 - "@docusaurus/theme-search-algolia": 0.0.0-5569 - "@docusaurus/types": 0.0.0-5569 + "@docusaurus/core": 0.0.0-5580 + "@docusaurus/plugin-content-blog": 0.0.0-5580 + "@docusaurus/plugin-content-docs": 0.0.0-5580 + "@docusaurus/plugin-content-pages": 0.0.0-5580 + "@docusaurus/plugin-debug": 0.0.0-5580 + "@docusaurus/plugin-google-analytics": 0.0.0-5580 + "@docusaurus/plugin-google-gtag": 0.0.0-5580 + "@docusaurus/plugin-google-tag-manager": 0.0.0-5580 + "@docusaurus/plugin-sitemap": 0.0.0-5580 + "@docusaurus/theme-classic": 0.0.0-5580 + "@docusaurus/theme-common": 0.0.0-5580 + "@docusaurus/theme-search-algolia": 0.0.0-5580 + "@docusaurus/types": 0.0.0-5580 peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: a20013db1bd1c77e6fc2b901326d62f3003b6f6e6191f26aae6968cc2402837c0e84cc54571849d2af811fdae74d9359c0e96775d61b9f3efbbe881898e7f9a3 + checksum: 3aa48aea40e33c2c7adcad54918e117b1f1d917eba110889d85a273e82b4bbf0d73d9e5bb8c5aaadd846ec341981bbe701014cbd93ddf55c83245ce1bc4a56a6 languageName: node linkType: hard @@ -1992,22 +1993,22 @@ __metadata: languageName: node linkType: hard -"@docusaurus/theme-classic@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/theme-classic@npm:0.0.0-5569" +"@docusaurus/theme-classic@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/theme-classic@npm:0.0.0-5580" dependencies: - "@docusaurus/core": 0.0.0-5569 - "@docusaurus/mdx-loader": 0.0.0-5569 - "@docusaurus/module-type-aliases": 0.0.0-5569 - "@docusaurus/plugin-content-blog": 0.0.0-5569 - "@docusaurus/plugin-content-docs": 0.0.0-5569 - "@docusaurus/plugin-content-pages": 0.0.0-5569 - "@docusaurus/theme-common": 0.0.0-5569 - "@docusaurus/theme-translations": 0.0.0-5569 - "@docusaurus/types": 0.0.0-5569 - "@docusaurus/utils": 0.0.0-5569 - "@docusaurus/utils-common": 0.0.0-5569 - "@docusaurus/utils-validation": 0.0.0-5569 + "@docusaurus/core": 0.0.0-5580 + "@docusaurus/mdx-loader": 0.0.0-5580 + "@docusaurus/module-type-aliases": 0.0.0-5580 + "@docusaurus/plugin-content-blog": 0.0.0-5580 + "@docusaurus/plugin-content-docs": 0.0.0-5580 + "@docusaurus/plugin-content-pages": 0.0.0-5580 + "@docusaurus/theme-common": 0.0.0-5580 + "@docusaurus/theme-translations": 0.0.0-5580 + "@docusaurus/types": 0.0.0-5580 + "@docusaurus/utils": 0.0.0-5580 + "@docusaurus/utils-common": 0.0.0-5580 + "@docusaurus/utils-validation": 0.0.0-5580 "@mdx-js/react": ^2.1.5 clsx: ^1.2.1 copy-text-to-clipboard: ^3.0.1 @@ -2024,21 +2025,21 @@ __metadata: peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: ee25082a4abc0d812cc54ee1bea642766f903cfa47d28c6a6efbc53cdaacf67bee594a4745c7ae97fe3ba6ddb136de36dd0c8cda2c310c5b40c9023bee70fcbb + checksum: 6a8f99d5143017d4754601fbf6487b697dc77945f730763917578abe0a1ef694607f888d824146aef86fdc2a620f0ddbd4925a23dcfc15b57d8338c1de73d526 languageName: node linkType: hard -"@docusaurus/theme-common@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/theme-common@npm:0.0.0-5569" +"@docusaurus/theme-common@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/theme-common@npm:0.0.0-5580" dependencies: - "@docusaurus/mdx-loader": 0.0.0-5569 - "@docusaurus/module-type-aliases": 0.0.0-5569 - "@docusaurus/plugin-content-blog": 0.0.0-5569 - "@docusaurus/plugin-content-docs": 0.0.0-5569 - "@docusaurus/plugin-content-pages": 0.0.0-5569 - "@docusaurus/utils": 0.0.0-5569 - "@docusaurus/utils-common": 0.0.0-5569 + "@docusaurus/mdx-loader": 0.0.0-5580 + "@docusaurus/module-type-aliases": 0.0.0-5580 + "@docusaurus/plugin-content-blog": 0.0.0-5580 + "@docusaurus/plugin-content-docs": 0.0.0-5580 + "@docusaurus/plugin-content-pages": 0.0.0-5580 + "@docusaurus/utils": 0.0.0-5580 + "@docusaurus/utils-common": 0.0.0-5580 "@types/history": ^4.7.11 "@types/react": "*" "@types/react-router-config": "*" @@ -2051,22 +2052,22 @@ __metadata: peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: 26a2d7dd6fd8b97e67db3bc093fcb9ce6f14d561aeccf94c3334d18a9a2900483dcb0ec4bb28d31e910bea459f20e4445207d5156c9cc02bd72601eb0256e3cc + checksum: 99524cb36a520568f63140734daaea8d56b35c0c8de3c36cb0102a93a0714f31a68aa5302607b9113a68e1b9fb7d73fd53ac80aa82ce48bf6f2d9e304ce21da2 languageName: node linkType: hard -"@docusaurus/theme-search-algolia@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/theme-search-algolia@npm:0.0.0-5569" +"@docusaurus/theme-search-algolia@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/theme-search-algolia@npm:0.0.0-5580" dependencies: "@docsearch/react": ^3.3.3 - "@docusaurus/core": 0.0.0-5569 - "@docusaurus/logger": 0.0.0-5569 - "@docusaurus/plugin-content-docs": 0.0.0-5569 - "@docusaurus/theme-common": 0.0.0-5569 - "@docusaurus/theme-translations": 0.0.0-5569 - "@docusaurus/utils": 0.0.0-5569 - "@docusaurus/utils-validation": 0.0.0-5569 + "@docusaurus/core": 0.0.0-5580 + "@docusaurus/logger": 0.0.0-5580 + "@docusaurus/plugin-content-docs": 0.0.0-5580 + "@docusaurus/theme-common": 0.0.0-5580 + "@docusaurus/theme-translations": 0.0.0-5580 + "@docusaurus/utils": 0.0.0-5580 + "@docusaurus/utils-validation": 0.0.0-5580 algoliasearch: ^4.15.0 algoliasearch-helper: ^3.12.0 clsx: ^1.2.1 @@ -2078,23 +2079,23 @@ __metadata: peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: 16178b7b9278c91eaedbb1629192dc2255939de6f509b721305c4c1795283e0fd4f859b305ce051ec5efdbe71884ceede6239445916b60ebec1858e3b5161d8f + checksum: efc6054ad979b9fde43bec8befeb41c0f9ca609832b51e4da42fdcdfd7e4ee599fd3e6949c576d9ff348c0ffd044a4dbb07638a1990f57586f493ba394369c96 languageName: node linkType: hard -"@docusaurus/theme-translations@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/theme-translations@npm:0.0.0-5569" +"@docusaurus/theme-translations@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/theme-translations@npm:0.0.0-5580" dependencies: fs-extra: ^11.1.0 tslib: ^2.5.0 - checksum: 695f94ddc66d54b649042f0fb0a88b692613d10898ee3306c6c9b52340373493ac9bcceec9263332454d7c9179339517e40b083e949510f9e48ec834bc61fbb3 + checksum: 1555956d2a7cf3122ded374937559ab19e6a6620fd0910d6bdbf48be78a10e595cfd9a0b9157c129f36f0bfae4297f07991f071a394c87cb9d2facee04597dd2 languageName: node linkType: hard -"@docusaurus/types@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/types@npm:0.0.0-5569" +"@docusaurus/types@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/types@npm:0.0.0-5580" dependencies: "@types/history": ^4.7.11 "@types/react": "*" @@ -2107,13 +2108,13 @@ __metadata: peerDependencies: react: ^16.8.4 || ^17.0.0 react-dom: ^16.8.4 || ^17.0.0 - checksum: 956b7af3a75b3632d1ff330d018353b38b22b3f4af8b39f452fcde33d74c70312dadd3e3bb40e04ecbe02e2462a67bfcbd4ef631462a9c710616c83186930b69 + checksum: 5df67f562c5d516eb020ecf63e6343e1d6d18e2716b1b530a24c5d63b48229623222861c1a69f0e4715c34a768f7c05599340f1af18ca1259709467fb1149af9 languageName: node linkType: hard -"@docusaurus/utils-common@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/utils-common@npm:0.0.0-5569" +"@docusaurus/utils-common@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/utils-common@npm:0.0.0-5580" dependencies: tslib: ^2.5.0 peerDependencies: @@ -2121,28 +2122,28 @@ __metadata: peerDependenciesMeta: "@docusaurus/types": optional: true - checksum: 7d98e31beea34be91de7fdc971033439e412f535617707c79c4d2c0653c769ff4ce7521bd87f038811ccbae3b1d5c0e5d26eab9935ee8bc3515b926cbe175365 + checksum: 238b2f12607834164fa82f9a3154703b75609719888cf96f06c9f70c17ab76a46babcfd66dce4f2722ecaaf47742a40a606101f0578f2345845fa23b6f262b5d languageName: node linkType: hard -"@docusaurus/utils-validation@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/utils-validation@npm:0.0.0-5569" +"@docusaurus/utils-validation@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/utils-validation@npm:0.0.0-5580" dependencies: - "@docusaurus/logger": 0.0.0-5569 - "@docusaurus/utils": 0.0.0-5569 + "@docusaurus/logger": 0.0.0-5580 + "@docusaurus/utils": 0.0.0-5580 joi: ^17.8.3 js-yaml: ^4.1.0 tslib: ^2.5.0 - checksum: b16730923e0d6fa627ca3e3105c152e814d540badb93f0a2ae1194672a76f8269ed48ed5c8f3259c0e0b9dfc6dc939035e0ac2e675b283ac4d414c642a95c893 + checksum: cf0602b2411738adc47f5596fd4a3076137aa754aca24a4630700dc6355fed07e9a66ec093e438fa1199d9076bebae036317d073a604e4f0666f8390ceb73b5d languageName: node linkType: hard -"@docusaurus/utils@npm:0.0.0-5569": - version: 0.0.0-5569 - resolution: "@docusaurus/utils@npm:0.0.0-5569" +"@docusaurus/utils@npm:0.0.0-5580": + version: 0.0.0-5580 + resolution: "@docusaurus/utils@npm:0.0.0-5580" dependencies: - "@docusaurus/logger": 0.0.0-5569 + "@docusaurus/logger": 0.0.0-5580 "@svgr/webpack": ^6.5.1 escape-string-regexp: ^4.0.0 file-loader: ^6.2.0 @@ -2163,7 +2164,7 @@ __metadata: peerDependenciesMeta: "@docusaurus/types": optional: true - checksum: 33e044f9b5328cf9d1b17637fa41a6138c5e18b5f3d73389bbb9f71735a273a9e27660d9a9103766406549edfa0691e2524cc84e6ebc2b0d308991df3bee0dd9 + checksum: c9b085d3724cba748b3db763f056742aace45cc58a636e23513ee6a7db781558377de5507b651b3566562fa71eebb42ac835034cdcbcc2c77edf9b4009c6a215 languageName: node linkType: hard @@ -2991,6 +2992,13 @@ __metadata: languageName: node linkType: hard +"@types/parse5@npm:^6.0.0": + version: 6.0.3 + resolution: "@types/parse5@npm:6.0.3" + checksum: ddb59ee4144af5dfcc508a8dcf32f37879d11e12559561e65788756b95b33e6f03ea027d88e1f5408f9b7bfb656bf630ace31a2169edf44151daaf8dd58df1b7 + languageName: node + linkType: hard + "@types/prop-types@npm:*": version: 15.7.5 resolution: "@types/prop-types@npm:15.7.5" @@ -3737,10 +3745,10 @@ __metadata: version: 0.0.0-use.local resolution: "backstage-microsite@workspace:." dependencies: - "@docusaurus/core": 0.0.0-5569 - "@docusaurus/module-type-aliases": 0.0.0-5569 - "@docusaurus/plugin-client-redirects": 0.0.0-5569 - "@docusaurus/preset-classic": 0.0.0-5569 + "@docusaurus/core": 0.0.0-5580 + "@docusaurus/module-type-aliases": 0.0.0-5580 + "@docusaurus/plugin-client-redirects": 0.0.0-5580 + "@docusaurus/preset-classic": 0.0.0-5580 "@spotify/prettier-config": ^14.0.0 "@swc/core": ^1.3.46 "@tsconfig/docusaurus": ^1.0.6 @@ -6132,6 +6140,21 @@ __metadata: languageName: node linkType: hard +"hast-util-from-parse5@npm:^7.0.0": + version: 7.1.2 + resolution: "hast-util-from-parse5@npm:7.1.2" + dependencies: + "@types/hast": ^2.0.0 + "@types/unist": ^2.0.0 + hastscript: ^7.0.0 + property-information: ^6.0.0 + vfile: ^5.0.0 + vfile-location: ^4.0.0 + web-namespaces: ^2.0.0 + checksum: 7b4ed5b508b1352127c6719f7b0c0880190cf9859fe54ccaf7c9228ecf623d36cef3097910b3874d2fe1aac6bf4cf45d3cc2303daac3135a05e9ade6534ddddb + languageName: node + linkType: hard + "hast-util-parse-selector@npm:^3.0.0": version: 3.1.1 resolution: "hast-util-parse-selector@npm:3.1.1" @@ -6141,6 +6164,25 @@ __metadata: languageName: node linkType: hard +"hast-util-raw@npm:^7.2.0": + version: 7.2.3 + resolution: "hast-util-raw@npm:7.2.3" + dependencies: + "@types/hast": ^2.0.0 + "@types/parse5": ^6.0.0 + hast-util-from-parse5: ^7.0.0 + hast-util-to-parse5: ^7.0.0 + html-void-elements: ^2.0.0 + parse5: ^6.0.0 + unist-util-position: ^4.0.0 + unist-util-visit: ^4.0.0 + vfile: ^5.0.0 + web-namespaces: ^2.0.0 + zwitch: ^2.0.0 + checksum: 21857eea3ffb8fd92d2d9be7793b56d0b2c40db03c4cfa14828855ae41d7c584917aa83efb7157220b2e41e25e95f81f24679ac342c35145e5f1c1d39015f81f + languageName: node + linkType: hard + "hast-util-to-estree@npm:^2.0.0": version: 2.3.2 resolution: "hast-util-to-estree@npm:2.3.2" @@ -6164,6 +6206,20 @@ __metadata: languageName: node linkType: hard +"hast-util-to-parse5@npm:^7.0.0": + version: 7.1.0 + resolution: "hast-util-to-parse5@npm:7.1.0" + dependencies: + "@types/hast": ^2.0.0 + comma-separated-tokens: ^2.0.0 + property-information: ^6.0.0 + space-separated-tokens: ^2.0.0 + web-namespaces: ^2.0.0 + zwitch: ^2.0.0 + checksum: 3a7f2175a3db599bbae7e49ba73d3e5e688e5efca7590ff50130ba108ad649f728402815d47db49146f6b94c14c934bf119915da9f6964e38802c122bcc8af6b + languageName: node + linkType: hard + "hast-util-whitespace@npm:^2.0.0": version: 2.0.1 resolution: "hast-util-whitespace@npm:2.0.1" @@ -6171,7 +6227,7 @@ __metadata: languageName: node linkType: hard -"hastscript@npm:^7.1.0": +"hastscript@npm:^7.0.0, hastscript@npm:^7.1.0": version: 7.2.0 resolution: "hastscript@npm:7.2.0" dependencies: @@ -6276,6 +6332,13 @@ __metadata: languageName: node linkType: hard +"html-void-elements@npm:^2.0.0": + version: 2.0.1 + resolution: "html-void-elements@npm:2.0.1" + checksum: 06d41f13b9d5d6e0f39861c4bec9a9196fa4906d56cd5cf6cf54ad2e52a85bf960cca2bf9600026bde16c8331db171bedba5e5a35e2e43630c8f1d497b2fb658 + languageName: node + linkType: hard + "html-webpack-plugin@npm:^5.5.0": version: 5.5.0 resolution: "html-webpack-plugin@npm:5.5.0" @@ -8735,6 +8798,13 @@ __metadata: languageName: node linkType: hard +"parse5@npm:^6.0.0": + version: 6.0.1 + resolution: "parse5@npm:6.0.1" + checksum: 7d569a176c5460897f7c8f3377eff640d54132b9be51ae8a8fa4979af940830b2b0c296ce75e5bd8f4041520aadde13170dbdec44889975f906098ea0002f4bd + languageName: node + linkType: hard + "parse5@npm:^7.0.0": version: 7.1.2 resolution: "parse5@npm:7.1.2" @@ -9901,6 +9971,17 @@ __metadata: languageName: node linkType: hard +"rehype-raw@npm:^6.1.1": + version: 6.1.1 + resolution: "rehype-raw@npm:6.1.1" + dependencies: + "@types/hast": ^2.0.0 + hast-util-raw: ^7.2.0 + unified: ^10.0.0 + checksum: a1f9d309e609f49fb1f1e06e722705f4dd2e569653a89f756eaccb33b612cf1bb511216a81d10a619d11d047afc161e4b3cb99b957df05a8ba8fdbd5843f949a + languageName: node + linkType: hard + "relateurl@npm:^0.2.7": version: 0.2.7 resolution: "relateurl@npm:0.2.7" @@ -11526,6 +11607,13 @@ __metadata: languageName: node linkType: hard +"web-namespaces@npm:^2.0.0": + version: 2.0.1 + resolution: "web-namespaces@npm:2.0.1" + checksum: b6d9f02f1a43d0ef0848a812d89c83801d5bbad57d8bb61f02eb6d7eb794c3736f6cc2e1191664bb26136594c8218ac609f4069722c6f56d9fc2d808fa9271c6 + languageName: node + linkType: hard + "webidl-conversions@npm:^3.0.0": version: 3.0.1 resolution: "webidl-conversions@npm:3.0.1" From d689fe34e37dbd5a53d2825c3d311fd3307d346b Mon Sep 17 00:00:00 2001 From: Samuel Date: Thu, 11 May 2023 18:39:07 +0100 Subject: [PATCH 24/29] docs: Highlight error handling instructions with a note Signed-off-by: Samuel --- docs/getting-started/configuration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index ddf50c64e6..87995746d8 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -242,7 +242,8 @@ integrations: - host: github.com token: ${GITHUB_TOKEN} # this will use the environment variable GITHUB_TOKEN ``` -> If you encounter the error `No token available for host: github.com, with owner , and repo from-backstage`, try resolving it by restarting Backstage. First, stop the running instance by pressing `Control-C` in the terminal. Then, restart it by running the command `yarn dev`. Once Backstage is up and running, click the **START OVER** button." + +> Note: If you've updated the configuration for your integration, it's likely that the backend will need a restart to apply these changes. To do this, stop the running instance in your terminal with `Control-C`, then start it again with `yarn dev`. Once the backend has restarted, retry the operation. Some helpful links, for if you want to learn more about: From 4c86ec146602a27d54013b71ea41890c99073fb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 16 May 2023 22:02:57 +0200 Subject: [PATCH 25/29] add release to microsite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .github/vale/Vocab/Backstage/accept.txt | 3 + docs/releases/v1.14.0.md | 80 +++++++++++++++++++++++++ microsite/docusaurus.config.js | 2 +- microsite/sidebars.json | 1 + 4 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 docs/releases/v1.14.0.md diff --git a/.github/vale/Vocab/Backstage/accept.txt b/.github/vale/Vocab/Backstage/accept.txt index 83f05e390d..68fbdabb00 100644 --- a/.github/vale/Vocab/Backstage/accept.txt +++ b/.github/vale/Vocab/Backstage/accept.txt @@ -316,6 +316,7 @@ scaffolded scaffolder Scaffolder scrollbar +sdks seb semlas semver @@ -343,6 +344,8 @@ stdout storable stringified stringify +subcommand +subcommands subcomponent subcomponents subfolder diff --git a/docs/releases/v1.14.0.md b/docs/releases/v1.14.0.md new file mode 100644 index 0000000000..2c565e2519 --- /dev/null +++ b/docs/releases/v1.14.0.md @@ -0,0 +1,80 @@ +--- +id: v1.14.0 +title: v1.14.0 +description: Backstage Release v1.14.0 +--- + +These are the release notes for the v1.14.0 release of [Backstage](https://backstage.io/). + +A huge thanks to the whole team of maintainers and contributors as well as the amazing Backstage Community for their hard work in getting this release developed and done. + +## Highlights + +This release has an important security fix, along with a lot of squashed bugs and exciting additions! Enjoy. + +### **BREAKING**: Tweaks to the OpenAPI tooling + +There’s been further work made on the OpenAPI tooling! One of the changes is that the commands for this feature are grouped under a `schema openapi` subcommand. This lets us structure things a bit better for the future. The generated file now also has a `.generated.ts` extension, to more easily be able to keep it apart in linting and similar. + +### **BREAKING**: Kubernetes plugin log viewer and tweaks + +There’s been some awesome additions made to the Kubernetes plugin, including the ability to show pod logs! To support this, some breaking changes were made in some of its interfaces. Check out [#17120](https://github.com/backstage/backstage/pull/17120) for some details of what changed. + +### **DEPRECATION**: React 16 + +We are preparing for React 18, and as part of doing that, we are now officially deprecating support for React 16. Nothing will break for you just yet, but if you are still on React 16 your `backstage-cli` commands will start to show helpful warning messages guiding you to bump to version 17 when you can. + +See [#17752](https://github.com/backstage/backstage/pull/17752) for some details about this, and links to the relevant issues. + +### New plugin: DevTools + +This plugin is focused on integrators rather than end users. It allows you to see useful information about your Backstage installation, such as what operating system and NodeJS version it’s running on, what Backstage and individual package versions you have, and more! This may serve as a foundation for adding even more ops focused features in the future. + +Contributed by [@awanlin](https://github.com/awanlin) in [#17393](https://github.com/backstage/backstage/pull/17393) + +### Migration to AWS-SDK version 3 + +There’s been work done under the hood to migrate AWS features over to using v3 of their client SDKs. While this should mostly go below the radar and ultimately work the same as it did before, do reach out to us if you encounter any problems that might be related to this migration. + +There’s actually a minor interface breakage as part of this, where `AwsIamKubernetesAuthTranslator` has some methods removed, but that’s one which may not have a lot of use outside of the package. + +Contributed by [@aochsner](https://github.com/aochsner), see PRs linked at the bottom of issue [#16470](https://github.com/backstage/backstage/issues/16470) + +### Persistent session store for the auth backend + +For auth providers that use session storage, those sessions are now persisted in the auth backend database, instead of in local memory. This should make session handling work better across scaled auth backend instances. + +### Markdown Output Support for Software Templates + +You can now output Markdown from the Software Template runs which can provide more data to the end user after a job has completed in the outputs section of `scaffolder/next` + +Contributed by [@voximity](https://github.com/voximity) in [#17641](https://github.com/backstage/backstage/pull/17641) + +### Refactored Configuration Loading + +The configuration loading system has been refactored to make it easier to extend the system with additional sources of configuration. See the [initial PR](https://github.com/backstage/backstage/pull/17209) and the [`@backstage/config-loader` changelog](https://github.com/backstage/backstage/blob/master/packages/config-loader/CHANGELOG.md#130) for more details. + +### Catalog database performance tweaks + +We found some hotspots in the catalog database operations, and streamlined them. Hopefully this should be noticeable in the form of slightly lower CPU and IOPS usage. + +## Security Fixes + +A [security flaw in the `vm2` package](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-32314) used by the Scaffolder plugin was patched by our automated security processes in [#17810](https://github.com/backstage/backstage/pull/17810). The fixed version of `vm2` was already covered by the required version range of the Scaffolder plugin, so adopters are able to address this vulnerability separately without upgrading Backstage itself, as part of their regular security processes. + +## Upgrade path + +We recommend that you keep your Backstage project up to date with this latest release. For more guidance on how to upgrade, check out the documentation for [keeping Backstage updated](https://backstage.io/docs/getting-started/keeping-backstage-updated). + +## Links and References + +Below you can find a list of links and references to help you learn about and start using this new release. + +- [Backstage official website](https://backstage.io/), [documentation](https://backstage.io/docs/), and [getting started guide](https://backstage.io/docs/getting-started/) +- [GitHub repository](https://github.com/backstage/backstage) +- Backstage's [versioning and support policy](https://backstage.io/docs/overview/versioning-policy) +- [Community Discord](https://discord.gg/backstage-687207715902193673) for discussions and support +- [Changelog](https://github.com/backstage/backstage/tree/master/docs/releases/v1.14.0-changelog.md) +- Backstage [Demos](https://backstage.io/demos), [Blog](https://backstage.io/blog), [Roadmap](https://backstage.io/docs/overview/roadmap) and [Plugins](https://backstage.io/plugins) + +Sign up for our [newsletter](https://mailchi.mp/spotify/backstage-community) if you want to be informed about what is happening in the world of Backstage. diff --git a/microsite/docusaurus.config.js b/microsite/docusaurus.config.js index d0f6bb277f..9985796e3f 100644 --- a/microsite/docusaurus.config.js +++ b/microsite/docusaurus.config.js @@ -173,7 +173,7 @@ module.exports = { position: 'left', }, { - to: 'docs/releases/v1.12.0', + to: 'docs/releases/v1.14.0', label: 'Releases', position: 'left', }, diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 1515c483b9..e75683f803 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -1,6 +1,7 @@ { "releases": { "Release Notes": [ + "releases/v1.14.0", "releases/v1.13.0", "releases/v1.12.0", "releases/v1.11.0", From 8b45bc7c122d6ec931d592373896518438864ffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 16 May 2023 22:46:02 +0200 Subject: [PATCH 26/29] adjust change logs to show enableExperimentalRedirectFlow at root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- docs/releases/v1.13.0-changelog.md | 10 +++++----- docs/releases/v1.13.0-next.0-changelog.md | 10 +++++----- docs/releases/v1.13.0.md | 2 +- packages/core-app-api/config.d.ts | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/releases/v1.13.0-changelog.md b/docs/releases/v1.13.0-changelog.md index 823ff337bc..506800eb38 100644 --- a/docs/releases/v1.13.0-changelog.md +++ b/docs/releases/v1.13.0-changelog.md @@ -4,7 +4,7 @@ ### Minor Changes -- 7908d72e033: Introduce a new global config parameter, `auth.enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. +- 7908d72e033: Introduce a new global config parameter, `enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. ### Patch Changes @@ -61,7 +61,7 @@ ### Minor Changes -- 7908d72e033: Introduce a new global config parameter, `auth.enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. +- 7908d72e033: Introduce a new global config parameter, `enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. - c15e0cedbe1: The `AuthConnector` interface now supports specifying a set of scopes when refreshing a session. The `DefaultAuthConnector` implementation passes the `scope` query parameter to the auth-backend plugin appropriately. The @@ -106,7 +106,7 @@ - 67140d9f96f: Upgrade `react-virtualized-auto-sizer´ to version `^1.0.11\` - 6e0b71493df: Switched internal declaration of `DependencyGraphTypes` to use `namespace`. - c8779cc1d09: Updated `LogLine` component, which is used by the `LogViewer`, to turn URLs into clickable links. This feature is on by default -- 7908d72e033: Introduce a new global config parameter, `auth.enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. +- 7908d72e033: Introduce a new global config parameter, `enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. - 1e4f5e91b8e: Bump `zod` and `zod-to-json-schema` dependencies. - 29ba8267d69: Updated dependency `@material-ui/lab` to `4.0.0-alpha.61`. - 8e00acb28db: Small tweaks to remove warnings in the console during development (mainly focusing on techdocs) @@ -170,7 +170,7 @@ ### Minor Changes -- 7908d72e033: Introduce a new global config parameter, `auth.enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. +- 7908d72e033: Introduce a new global config parameter, `enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. ### Patch Changes @@ -1245,7 +1245,7 @@ - d8f774c30df: Enforce the secret visibility of certificates and client secrets in the auth backend. Also, document all known options for each auth plugin. -- 7908d72e033: Introduce a new global config parameter, `auth.enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. +- 7908d72e033: Introduce a new global config parameter, `enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. - 475abd1dc3f: The `microsoft` (i.e. Azure) auth provider now supports negotiating tokens for Azure resources besides Microsoft Graph (e.g. AKS, Virtual Machines, Machine diff --git a/docs/releases/v1.13.0-next.0-changelog.md b/docs/releases/v1.13.0-next.0-changelog.md index 3075d80f61..7de0a370f9 100644 --- a/docs/releases/v1.13.0-next.0-changelog.md +++ b/docs/releases/v1.13.0-next.0-changelog.md @@ -4,7 +4,7 @@ ### Minor Changes -- 7908d72e033: Introduce a new global config parameter, `auth.enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. +- 7908d72e033: Introduce a new global config parameter, `enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. ### Patch Changes @@ -19,7 +19,7 @@ ### Minor Changes -- 7908d72e033: Introduce a new global config parameter, `auth.enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. +- 7908d72e033: Introduce a new global config parameter, `enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. ### Patch Changes @@ -33,7 +33,7 @@ ### Minor Changes -- 7908d72e033: Introduce a new global config parameter, `auth.enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. +- 7908d72e033: Introduce a new global config parameter, `enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. ### Patch Changes @@ -336,7 +336,7 @@ ### Patch Changes -- 7908d72e033: Introduce a new global config parameter, `auth.enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. +- 7908d72e033: Introduce a new global config parameter, `enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. - 8e00acb28db: Small tweaks to remove warnings in the console during development (mainly focusing on techdocs) - 7245e744ab1: Fixed the font color on `BackstageHeaderLabel` to respect the active page theme. - Updated dependencies @@ -530,7 +530,7 @@ ### Patch Changes - d8f774c30df: Enforce the secret visibility of certificates and client secrets in the auth backend. Also, document all known options for each auth plugin. -- 7908d72e033: Introduce a new global config parameter, `auth.enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. +- 7908d72e033: Introduce a new global config parameter, `enableExperimentalRedirectFlow`. When enabled, auth will happen with an in-window redirect flow rather than through a popup window. - Updated dependencies - @backstage/backend-common@0.18.4-next.0 - @backstage/config@1.0.7 diff --git a/docs/releases/v1.13.0.md b/docs/releases/v1.13.0.md index 99253ac3d2..c8803453d4 100644 --- a/docs/releases/v1.13.0.md +++ b/docs/releases/v1.13.0.md @@ -86,7 +86,7 @@ Contributed by [@sennyeya](https://github.com/sennyeya) in [#15667](https://gith ### Experimental inline auth flows -There’s new support for running auth flows inline in the current page instead of in a separate popup window, enabled via the `auth.enableExperimentalRedirectFlow` app-config parameter. This will allow the use of some providers that do not support popup flows. It’s still an experimental feature, but do try it out if this applies to you! +There’s new support for running auth flows inline in the current page instead of in a separate popup window, enabled via the `enableExperimentalRedirectFlow` app-config parameter. This will allow the use of some providers that do not support popup flows. It’s still an experimental feature, but do try it out if this applies to you! Contributed by [@headphonejames](https://github.com/headphonejames) in [#15841](https://github.com/backstage/backstage/pull/15841) diff --git a/packages/core-app-api/config.d.ts b/packages/core-app-api/config.d.ts index 2de1d96114..e8cc879c90 100644 --- a/packages/core-app-api/config.d.ts +++ b/packages/core-app-api/config.d.ts @@ -117,8 +117,8 @@ export interface Config { }; /** - $ Enable redirect authentication flow type, instead of a popup for authentication - * default value: 'false' + * Enable redirect authentication flow type, instead of a popup for authentication. + * @defaultValue false * @visibility frontend */ enableExperimentalRedirectFlow?: boolean; From 2ebf6eaf82d080313e064d2c6d6585456bee8f42 Mon Sep 17 00:00:00 2001 From: Christophe Fargette Date: Tue, 16 May 2023 23:48:04 -0400 Subject: [PATCH 27/29] Add 7 Janus plugins Signed-off-by: Christophe Fargette --- microsite/data/plugins/3scale.yaml | 10 ++++++++++ microsite/data/plugins/jfrog-artifactory.yaml | 10 ++++++++++ microsite/data/plugins/keycloak.yaml | 10 ++++++++++ microsite/data/plugins/ocm.yaml | 10 ++++++++++ microsite/data/plugins/quay.yaml | 10 ++++++++++ microsite/data/plugins/tekton.yaml | 10 ++++++++++ microsite/data/plugins/topology.yaml | 10 ++++++++++ 7 files changed, 70 insertions(+) create mode 100644 microsite/data/plugins/3scale.yaml create mode 100644 microsite/data/plugins/jfrog-artifactory.yaml create mode 100644 microsite/data/plugins/keycloak.yaml create mode 100644 microsite/data/plugins/ocm.yaml create mode 100644 microsite/data/plugins/quay.yaml create mode 100644 microsite/data/plugins/tekton.yaml create mode 100644 microsite/data/plugins/topology.yaml diff --git a/microsite/data/plugins/3scale.yaml b/microsite/data/plugins/3scale.yaml new file mode 100644 index 0000000000..c190ed7e9e --- /dev/null +++ b/microsite/data/plugins/3scale.yaml @@ -0,0 +1,10 @@ +--- +title: APIs with 3scale +author: Red Hat +authorUrl: https://redhat.com +category: Discovery +description: Synchronize 3scale content into the Backstage catalog. +documentation: https://janus-idp.io/plugins/3scale +iconUrl: http://janus-idp.io/images/plugins/3scale.svg +npmPackageName: '@janus-idp/backstage-plugin-3scale-backend' +addedDate: '2023-05-15' diff --git a/microsite/data/plugins/jfrog-artifactory.yaml b/microsite/data/plugins/jfrog-artifactory.yaml new file mode 100644 index 0000000000..279cbc161e --- /dev/null +++ b/microsite/data/plugins/jfrog-artifactory.yaml @@ -0,0 +1,10 @@ +--- +title: Container Image Registry for JFrog Artifactory +author: Red Hat +authorUrl: https://redhat.com +category: Image +description: View container image details from JFrog Artifactory in Backstage. +documentation: https://janus-idp.io/plugins/jfrog-artifactory +iconUrl: http://janus-idp.io/images/plugins/jfrog-artifactory.svg +npmPackageName: '@janus-idp/backstage-plugin-jfrog-artifactory' +addedDate: '2023-05-15' diff --git a/microsite/data/plugins/keycloak.yaml b/microsite/data/plugins/keycloak.yaml new file mode 100644 index 0000000000..882d43f515 --- /dev/null +++ b/microsite/data/plugins/keycloak.yaml @@ -0,0 +1,10 @@ +--- +title: Authentication and Authorization with Keycloak +author: Red Hat +authorUrl: https://redhat.com +category: Authentication +description: Load users and groups from Keycloak, enabling use of multiple authentication providers to be applied to Backstage entities. +documentation: https://janus-idp.io/plugins/keycloak +iconUrl: http://janus-idp.io/images/plugins/keycloak.svg +npmPackageName: '@janus-idp/backstage-plugin-keycloak-backend' +addedDate: '2023-05-15' diff --git a/microsite/data/plugins/ocm.yaml b/microsite/data/plugins/ocm.yaml new file mode 100644 index 0000000000..15c23a0fb7 --- /dev/null +++ b/microsite/data/plugins/ocm.yaml @@ -0,0 +1,10 @@ +--- +title: Multi Cluster View with OCM +author: Red Hat +authorUrl: https://redhat.com +category: OCM +description: View clusters from OCM's MultiClusterHub and MultiCluster Engine in Backstage. +documentation: https://janus-idp.io/plugins/ocm +iconUrl: http://janus-idp.io/images/plugins/ocm.svg +npmPackageName: '@janus-idp/backstage-plugin-ocm' +addedDate: '2023-05-15' diff --git a/microsite/data/plugins/quay.yaml b/microsite/data/plugins/quay.yaml new file mode 100644 index 0000000000..e4aeb1ec1a --- /dev/null +++ b/microsite/data/plugins/quay.yaml @@ -0,0 +1,10 @@ +--- +title: Container Image Registry for Quay +author: Red Hat +authorUrl: https://redhat.com +category: Image +description: View container image details from Quay in Backstage. +documentation: https://janus-idp.io/plugins/quay +iconUrl: http://janus-idp.io/images/plugins/quay.svg +npmPackageName: '@janus-idp/backstage-plugin-quay' +addedDate: '2023-05-15' diff --git a/microsite/data/plugins/tekton.yaml b/microsite/data/plugins/tekton.yaml new file mode 100644 index 0000000000..b1ebfb1349 --- /dev/null +++ b/microsite/data/plugins/tekton.yaml @@ -0,0 +1,10 @@ +--- +title: Pipelines with Tekton +author: Red Hat +authorUrl: https://redhat.com +category: CI +description: Easily view Tekton PipelineRun status for your services in Backstage. +documentation: https://janus-idp.io/plugins/tekton +iconUrl: http://janus-idp.io/images/plugins/tekton.svg +npmPackageName: '@janus-idp/backstage-plugin-tekton' +addedDate: '2023-05-15' diff --git a/microsite/data/plugins/topology.yaml b/microsite/data/plugins/topology.yaml new file mode 100644 index 0000000000..2f26f7c755 --- /dev/null +++ b/microsite/data/plugins/topology.yaml @@ -0,0 +1,10 @@ +--- +title: Application Topology for Kubernetes +author: Red Hat +authorUrl: https://redhat.com +category: Kubernetes +description: Visualize the deployment status and related resources of your applications deployed on any Kubernetes cluster. +documentation: https://janus-idp.io/plugins/topology +iconUrl: http://janus-idp.io/images/plugins/topology.svg +npmPackageName: '@janus-idp/backstage-plugin-topology' +addedDate: '2023-05-15' From 65b63dba5c1abdc4d8ce4f4ab53125d9f0102c3b Mon Sep 17 00:00:00 2001 From: Sergey Shevchenko Date: Wed, 17 May 2023 10:58:25 +0300 Subject: [PATCH 28/29] docs: Fix authenticate-api-requests doc Signed-off-by: Sergey Shevchenko --- contrib/docs/tutorials/authenticate-api-requests.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/contrib/docs/tutorials/authenticate-api-requests.md b/contrib/docs/tutorials/authenticate-api-requests.md index 49b6ae0a4d..c14eaebb54 100644 --- a/contrib/docs/tutorials/authenticate-api-requests.md +++ b/contrib/docs/tutorials/authenticate-api-requests.md @@ -82,10 +82,20 @@ export const createAuthMiddleware = async ( }; ``` +Install cookie-parser: + +```bash +# From your Backstage root directory +yarn add --cwd packages/backend cookie-parser +``` + +Update routes in `packages/backend/src/index.ts`: + ```typescript // packages/backend/src/index.ts from a create-app deployment import { createAuthMiddleware } from './authMiddleware'; +import cookieParser from 'cookie-parser'; // ... From 84a5c7724c7e35a80f0d7f9201fae9d0da171391 Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Fri, 5 May 2023 19:27:59 -0300 Subject: [PATCH 29/29] fix: corrected the disconnection behavior to get stream logs Close #15002 Signed-off-by: Rogerio Angeliski --- .changeset/chatty-ads-attack.md | 5 + .../src/hooks/useEventStream.ts | 130 ++++++++++-------- 2 files changed, 79 insertions(+), 56 deletions(-) create mode 100644 .changeset/chatty-ads-attack.md diff --git a/.changeset/chatty-ads-attack.md b/.changeset/chatty-ads-attack.md new file mode 100644 index 0000000000..709db946dd --- /dev/null +++ b/.changeset/chatty-ads-attack.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-react': patch +--- + +fixed refresh problem when backstage backend disconnects without any feedback to user. Now we send a generic message and try to reconnect after 15 seconds diff --git a/plugins/scaffolder-react/src/hooks/useEventStream.ts b/plugins/scaffolder-react/src/hooks/useEventStream.ts index ec49911471..ad802dba99 100644 --- a/plugins/scaffolder-react/src/hooks/useEventStream.ts +++ b/plugins/scaffolder-react/src/hooks/useEventStream.ts @@ -169,67 +169,85 @@ export const useTaskEventStream = (taskId: string): TaskStream => { let didCancel = false; let subscription: Subscription | undefined; let logPusher: NodeJS.Timeout | undefined; - - scaffolderApi.getTask(taskId).then( - task => { - if (didCancel) { - return; - } - dispatch({ type: 'INIT', data: task }); - - // TODO(blam): Use a normal fetch to fetch the current log for the event stream - // and use that for an INIT_EVENTs dispatch event, and then - // use the last event ID to subscribe using after option to - // stream logs. Without this, if you have a lot of logs, it can look like the - // task is being rebuilt on load as it progresses through the steps at a slower - // rate whilst it builds the status from the event logs - const observable = scaffolderApi.streamLogs({ taskId }); - - const collectedLogEvents = new Array(); - - function emitLogs() { - if (collectedLogEvents.length) { - const logs = collectedLogEvents.splice( - 0, - collectedLogEvents.length, - ); - dispatch({ type: 'LOGS', data: logs }); + let retryCount = 1; + const startStreamLogProcess = () => + scaffolderApi.getTask(taskId).then( + task => { + if (didCancel) { + return; } - } + dispatch({ type: 'INIT', data: task }); - logPusher = setInterval(emitLogs, 500); + // TODO(blam): Use a normal fetch to fetch the current log for the event stream + // and use that for an INIT_EVENTs dispatch event, and then + // use the last event ID to subscribe using after option to + // stream logs. Without this, if you have a lot of logs, it can look like the + // task is being rebuilt on load as it progresses through the steps at a slower + // rate whilst it builds the status from the event logs + const observable = scaffolderApi.streamLogs({ taskId }); - subscription = observable.subscribe({ - next: event => { - switch (event.type) { - case 'log': - return collectedLogEvents.push(event); - case 'cancelled': - dispatch({ type: 'CANCELLED' }); - return undefined; - case 'completion': - emitLogs(); - dispatch({ type: 'COMPLETED', data: event }); - return undefined; - default: - throw new Error( - `Unhandled event type ${event.type} in observer`, - ); + const collectedLogEvents = new Array(); + + function emitLogs() { + if (collectedLogEvents.length) { + const logs = collectedLogEvents.splice( + 0, + collectedLogEvents.length, + ); + dispatch({ type: 'LOGS', data: logs }); } - }, - error: error => { - emitLogs(); - dispatch({ type: 'ERROR', data: error }); - }, - }); - }, - error => { - if (!didCancel) { - dispatch({ type: 'ERROR', data: error }); - } - }, - ); + } + logPusher = setInterval(emitLogs, 500); + + subscription = observable.subscribe({ + next: event => { + switch (event.type) { + case 'log': + return collectedLogEvents.push(event); + case 'cancelled': + dispatch({ type: 'CANCELLED' }); + return undefined; + case 'completion': + emitLogs(); + dispatch({ type: 'COMPLETED', data: event }); + return undefined; + default: + throw new Error( + `Unhandled event type ${event.type} in observer`, + ); + } + }, + error: error => { + emitLogs(); + // in some cases the error is a refused connection from backend + // this can happen from internet issues or proxy problems + // so we try to reconnect again after some time + // just to restart the fetch process + // details here https://github.com/backstage/backstage/issues/15002 + + if (!error.message) { + error.message = `We cannot connect at the moment, trying again in some seconds... Retrying (${retryCount}/3 retries)`; + } + + if (retryCount <= 3) { + setTimeout(() => { + retryCount += 1; + startStreamLogProcess(); + }, 15000); + } + + dispatch({ type: 'ERROR', data: error }); + }, + }); + }, + error => { + if (!didCancel) { + dispatch({ type: 'ERROR', data: error }); + } + }, + ); + startStreamLogProcess(); return () => { didCancel = true; if (subscription) {