From 42addd8c8df87a0a93e9ae824acdf11f43e55030 Mon Sep 17 00:00:00 2001 From: Alex Rybchenko Date: Fri, 18 Nov 2022 14:16:58 +0100 Subject: [PATCH 001/237] added formData to validation function Signed-off-by: Alex Rybchenko --- plugins/scaffolder/api-report.md | 9 +++++---- plugins/scaffolder/src/extensions/types.ts | 3 ++- .../TemplateWizardPage/Stepper/createAsyncValidators.ts | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index 33a5f8c2fe..f94cb26d21 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -110,9 +110,9 @@ export type EntityPickerUiOptions = export const EntityTagsPickerFieldExtension: FieldExtensionComponent< string[], { - showCounts?: boolean | undefined; - kinds?: string[] | undefined; helperText?: string | undefined; + kinds?: string[] | undefined; + showCounts?: boolean | undefined; } >; @@ -120,9 +120,9 @@ export const EntityTagsPickerFieldExtension: FieldExtensionComponent< export const EntityTagsPickerFieldSchema: FieldSchema< string[], { - showCounts?: boolean | undefined; - kinds?: string[] | undefined; helperText?: string | undefined; + kinds?: string[] | undefined; + showCounts?: boolean | undefined; } >; @@ -224,6 +224,7 @@ export type NextCustomFieldValidator = ( field: FieldValidation_2, context: { apiHolder: ApiHolder; + formData: JsonObject; }, ) => void | Promise; diff --git a/plugins/scaffolder/src/extensions/types.ts b/plugins/scaffolder/src/extensions/types.ts index 3834b39c0a..5e133c676b 100644 --- a/plugins/scaffolder/src/extensions/types.ts +++ b/plugins/scaffolder/src/extensions/types.ts @@ -23,6 +23,7 @@ import { } from '@rjsf/utils'; import { PropsWithChildren } from 'react'; import { JSONSchema7 } from 'json-schema'; +import { JsonObject } from '@backstage/types'; /** * Field validation type for Custom Field Extensions. @@ -100,7 +101,7 @@ export interface NextFieldExtensionComponentProps< export type NextCustomFieldValidator = ( data: TFieldReturnValue, field: FieldValidationV5, - context: { apiHolder: ApiHolder }, + context: { apiHolder: ApiHolder; formData: JsonObject }, ) => void | Promise; /** diff --git a/plugins/scaffolder/src/next/TemplateWizardPage/Stepper/createAsyncValidators.ts b/plugins/scaffolder/src/next/TemplateWizardPage/Stepper/createAsyncValidators.ts index 1355387eda..6483ce2393 100644 --- a/plugins/scaffolder/src/next/TemplateWizardPage/Stepper/createAsyncValidators.ts +++ b/plugins/scaffolder/src/next/TemplateWizardPage/Stepper/createAsyncValidators.ts @@ -42,7 +42,7 @@ export const createAsyncValidators = ( if (validator) { const fieldValidation = createFieldValidation(); try { - await validator(value, fieldValidation, context); + await validator(value, fieldValidation, { ...context, formData }); } catch (ex) { fieldValidation.addError(ex.message); } From 596b7cf1a00729b06b80e967e6d2e0e834e213e9 Mon Sep 17 00:00:00 2001 From: Alex Rybchenko Date: Fri, 18 Nov 2022 14:47:10 +0100 Subject: [PATCH 002/237] initial support of ui options in scaffolder forms Signed-off-by: Alex Rybchenko --- .../sample-templates/bitbucket-demo/template.yaml | 2 ++ plugins/scaffolder-backend/src/service/router.ts | 1 + plugins/scaffolder/api-report.md | 3 +++ .../scaffolder/src/components/TemplatePage/TemplatePage.tsx | 1 + plugins/scaffolder/src/types.ts | 3 +++ 5 files changed, 10 insertions(+) diff --git a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml index 52714f6d73..269c3be1a6 100644 --- a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml @@ -4,6 +4,8 @@ metadata: name: bitbucket-demo title: Test Bitbucket RepoUrlPicker template description: scaffolder v1beta3 template demo publishing to bitbucket + ui:options: + finishButtonLabel: Publish spec: owner: backstage/techdocs-core type: service diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index eff7e95c50..be3c89601c 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -259,6 +259,7 @@ export async function createRouter( res.json({ title: template.metadata.title ?? template.metadata.name, description: template.metadata.description, + 'ui:options': template.metadata['ui:options'], steps: parameters.map(schema => ({ title: schema.title ?? 'Please enter the following information', description: schema.description, diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index 33a5f8c2fe..e9cbc5bb0e 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -640,6 +640,9 @@ export type TemplateGroupFilter = { export type TemplateParameterSchema = { title: string; description?: string; + ['ui:options']?: { + finishButtonLabel?: string; + }; steps: Array<{ title: string; description?: string; diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index 87678d2a83..4b2de4b56e 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -152,6 +152,7 @@ export const TemplatePage = ({ onReset={handleFormReset} onFinish={handleCreate} layouts={layouts} + finishButtonLabel={schema['ui:options']?.finishButtonLabel} steps={schema.steps.map(step => { return { ...step, diff --git a/plugins/scaffolder/src/types.ts b/plugins/scaffolder/src/types.ts index acb0f0e114..8645fc8697 100644 --- a/plugins/scaffolder/src/types.ts +++ b/plugins/scaffolder/src/types.ts @@ -80,6 +80,9 @@ export type ScaffolderTaskOutput = { export type TemplateParameterSchema = { title: string; description?: string; + ['ui:options']?: { + finishButtonLabel?: string; + }; steps: Array<{ title: string; description?: string; From 614083ef98be7297e3f4cf952e8f4f84acfdca13 Mon Sep 17 00:00:00 2001 From: Esther Annorzie Date: Mon, 21 Nov 2022 15:45:43 -0500 Subject: [PATCH 003/237] Add default message prop to StarredEntities Signed-off-by: Esther Annorzie --- .../homePageComponents/StarredEntities/Content.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/plugins/home/src/homePageComponents/StarredEntities/Content.tsx b/plugins/home/src/homePageComponents/StarredEntities/Content.tsx index 31fa00378e..aa0b6a5b72 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/Content.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/Content.tsx @@ -41,7 +41,14 @@ import useAsync from 'react-use/lib/useAsync'; * * @public */ -export const Content = () => { + +interface starredEntitiesProp { + defaultMessage?: string; +} + +export const Content = ({ + defaultMessage = 'Click the star beside an entity name to add the entity to this list!', +}: starredEntitiesProp) => { const catalogApi = useApi(catalogApiRef); const catalogEntityRoute = useRouteRef(entityRouteRef); const { starredEntities, toggleStarredEntity } = useStarredEntities(); @@ -76,7 +83,8 @@ export const Content = () => { if (starredEntities.size === 0) return ( - You do not have any starred entities yet! + {/* You do not have any starred entities yet! */} + {defaultMessage} ); From 9239c8acb87d2d95ca63f565208b4b03f07fa7c0 Mon Sep 17 00:00:00 2001 From: Esther Annorzie Date: Tue, 22 Nov 2022 10:09:55 -0500 Subject: [PATCH 004/237] Rename prop in starredEntitiesProp and remove comment Signed-off-by: Esther Annorzie --- .../src/homePageComponents/StarredEntities/Content.tsx | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/plugins/home/src/homePageComponents/StarredEntities/Content.tsx b/plugins/home/src/homePageComponents/StarredEntities/Content.tsx index aa0b6a5b72..1f31ac062f 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/Content.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/Content.tsx @@ -43,12 +43,10 @@ import useAsync from 'react-use/lib/useAsync'; */ interface starredEntitiesProp { - defaultMessage?: string; + noStarredEntitiesMessage?: React.ReactNode; } -export const Content = ({ - defaultMessage = 'Click the star beside an entity name to add the entity to this list!', -}: starredEntitiesProp) => { +export const Content = ({ noStarredEntitiesMessage }: starredEntitiesProp) => { const catalogApi = useApi(catalogApiRef); const catalogEntityRoute = useRouteRef(entityRouteRef); const { starredEntities, toggleStarredEntity } = useStarredEntities(); @@ -83,8 +81,8 @@ export const Content = ({ if (starredEntities.size === 0) return ( - {/* You do not have any starred entities yet! */} - {defaultMessage} + {noStarredEntitiesMessage || + 'Click the star beside an entity name to add it to this list!'} ); From b5e28f94fb8309b162fd50ecece32b937cd73a2b Mon Sep 17 00:00:00 2001 From: Esther Annorzie Date: Tue, 22 Nov 2022 18:05:10 -0500 Subject: [PATCH 005/237] Test call to action message when no entities are starred Signed-off-by: Esther Annorzie --- .../StarredEntities/Content.test.tsx | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx b/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx index 2b9fae8221..b761e89e6e 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx @@ -82,4 +82,34 @@ describe('StarredEntitiesContent', () => { '/catalog/default/component/mock-starred-entity-2', ); }); + + it('should display call to action message if no entities are starred', async () => { + const mockedApi = new MockStarredEntitiesApi(); + + const mockCatalogApi = { + getEntities: jest + .fn() + .mockImplementation(async () => ({ items: entities })), + }; + + const { getByText } = await renderInTestApp( + + + , + { + mountedRoutes: { + '/catalog/:namespace/:kind/:name': entityRouteRef, + }, + }, + ); + + expect( + getByText('Click the star beside an entity name to add it to this list!'), + ).toBeInTheDocument(); + }); }); From 777d6c7bdce0f1f741e32e6928a3caf0fe873873 Mon Sep 17 00:00:00 2001 From: Alex Rybchenko Date: Wed, 23 Nov 2022 15:53:36 +0100 Subject: [PATCH 006/237] leave only backend changes Signed-off-by: Alex Rybchenko --- .../sample-templates/bitbucket-demo/template.yaml | 2 -- plugins/scaffolder/api-report.md | 3 --- .../scaffolder/src/components/TemplatePage/TemplatePage.tsx | 1 - plugins/scaffolder/src/types.ts | 3 --- 4 files changed, 9 deletions(-) diff --git a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml index 269c3be1a6..52714f6d73 100644 --- a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml @@ -4,8 +4,6 @@ metadata: name: bitbucket-demo title: Test Bitbucket RepoUrlPicker template description: scaffolder v1beta3 template demo publishing to bitbucket - ui:options: - finishButtonLabel: Publish spec: owner: backstage/techdocs-core type: service diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index e9cbc5bb0e..33a5f8c2fe 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -640,9 +640,6 @@ export type TemplateGroupFilter = { export type TemplateParameterSchema = { title: string; description?: string; - ['ui:options']?: { - finishButtonLabel?: string; - }; steps: Array<{ title: string; description?: string; diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index 4b2de4b56e..87678d2a83 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -152,7 +152,6 @@ export const TemplatePage = ({ onReset={handleFormReset} onFinish={handleCreate} layouts={layouts} - finishButtonLabel={schema['ui:options']?.finishButtonLabel} steps={schema.steps.map(step => { return { ...step, diff --git a/plugins/scaffolder/src/types.ts b/plugins/scaffolder/src/types.ts index 8645fc8697..acb0f0e114 100644 --- a/plugins/scaffolder/src/types.ts +++ b/plugins/scaffolder/src/types.ts @@ -80,9 +80,6 @@ export type ScaffolderTaskOutput = { export type TemplateParameterSchema = { title: string; description?: string; - ['ui:options']?: { - finishButtonLabel?: string; - }; steps: Array<{ title: string; description?: string; From b07ccffad01f34abefa9cb3e5b313a9ab5a23e47 Mon Sep 17 00:00:00 2001 From: Alex Rybchenko Date: Wed, 23 Nov 2022 16:01:22 +0100 Subject: [PATCH 007/237] added changeset Signed-off-by: Alex Rybchenko --- .changeset/nasty-lizards-train.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/nasty-lizards-train.md diff --git a/.changeset/nasty-lizards-train.md b/.changeset/nasty-lizards-train.md new file mode 100644 index 0000000000..82d49cf682 --- /dev/null +++ b/.changeset/nasty-lizards-train.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Backend now returns 'ui:options' value from template metadata, it can be used by all your custom scaffolder components. From bef58bf44210fef30b3a4f2b4cea79ce1176c203 Mon Sep 17 00:00:00 2001 From: Alex Rybchenko Date: Fri, 18 Nov 2022 14:55:29 +0100 Subject: [PATCH 008/237] api report Signed-off-by: Alex Rybchenko --- plugins/scaffolder/api-report.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index f94cb26d21..46b2ea3c8a 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -110,9 +110,9 @@ export type EntityPickerUiOptions = export const EntityTagsPickerFieldExtension: FieldExtensionComponent< string[], { - helperText?: string | undefined; - kinds?: string[] | undefined; showCounts?: boolean | undefined; + kinds?: string[] | undefined; + helperText?: string | undefined; } >; @@ -120,9 +120,9 @@ export const EntityTagsPickerFieldExtension: FieldExtensionComponent< export const EntityTagsPickerFieldSchema: FieldSchema< string[], { - helperText?: string | undefined; - kinds?: string[] | undefined; showCounts?: boolean | undefined; + kinds?: string[] | undefined; + helperText?: string | undefined; } >; From 9000952e872d9ea5b27a5b09afe47722657d87ae Mon Sep 17 00:00:00 2001 From: Alex Rybchenko Date: Tue, 29 Nov 2022 17:55:10 +0100 Subject: [PATCH 009/237] added changeset Signed-off-by: Alex Rybchenko --- .changeset/nasty-dragons-melt.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/nasty-dragons-melt.md diff --git a/.changeset/nasty-dragons-melt.md b/.changeset/nasty-dragons-melt.md new file mode 100644 index 0000000000..b083ebce62 --- /dev/null +++ b/.changeset/nasty-dragons-melt.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': minor +--- + +All form data is now passed to validator functions in 'next' scaffolder, so it's now possible to perform validation for fields that depend on other field values From bfe38c80ea6202fc6233cbb0ee497762148caff1 Mon Sep 17 00:00:00 2001 From: Esther Annorzie Date: Tue, 29 Nov 2022 13:11:27 -0500 Subject: [PATCH 010/237] Add changeset Signed-off-by: Esther Annorzie --- .changeset/many-mangos-behave.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/many-mangos-behave.md diff --git a/.changeset/many-mangos-behave.md b/.changeset/many-mangos-behave.md new file mode 100644 index 0000000000..ff777d3839 --- /dev/null +++ b/.changeset/many-mangos-behave.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-home': minor +--- + +'backstage/home': minor + +If no entities are starred, a call to action message displays. From dbd2480a1a7b6d1489e34b3a755ad751cbc84dd1 Mon Sep 17 00:00:00 2001 From: Esther Annorzie Date: Wed, 30 Nov 2022 12:22:18 -0500 Subject: [PATCH 011/237] Test user provided message if no entities are starred Signed-off-by: Esther Annorzie --- .../StarredEntities/Content.test.tsx | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx b/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx index b761e89e6e..695406aab5 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx @@ -112,4 +112,32 @@ describe('StarredEntitiesContent', () => { getByText('Click the star beside an entity name to add it to this list!'), ).toBeInTheDocument(); }); + + it('should display user provided message if no entities are starred', async () => { + const mockedApi = new MockStarredEntitiesApi(); + + const mockCatalogApi = { + getEntities: jest + .fn() + .mockImplementation(async () => ({ items: entities })), + }; + + const { getByText } = await renderInTestApp( + + + , + { + mountedRoutes: { + '/catalog/:namespace/:kind/:name': entityRouteRef, + }, + }, + ); + + expect(getByText('foo')).toBeInTheDocument(); + }); }); From e48fc1f1ae82672151f8ebf1f85de34072693812 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 29 Nov 2022 14:22:47 +0100 Subject: [PATCH 012/237] Add optional, backward compatible logger to PG engine/indexer Signed-off-by: Eric Peterson --- .changeset/search-antibacterial-wipe.md | 12 ++++++++++++ plugins/search-backend-module-pg/api-report.md | 6 +++++- plugins/search-backend-module-pg/package.json | 3 ++- .../src/PgSearchEngine/PgSearchEngine.ts | 14 +++++++++++++- .../src/PgSearchEngine/PgSearchEngineIndexer.ts | 5 +++++ yarn.lock | 1 + 6 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 .changeset/search-antibacterial-wipe.md diff --git a/.changeset/search-antibacterial-wipe.md b/.changeset/search-antibacterial-wipe.md new file mode 100644 index 0000000000..d69edabd3f --- /dev/null +++ b/.changeset/search-antibacterial-wipe.md @@ -0,0 +1,12 @@ +--- +'@backstage/plugin-search-backend-module-pg': minor +--- + +Added the option to pass a logger to `PgSearchEngine` during instantiation. You may do so as follows: + +```diff +const searchEngine = await PgSearchEngine.fromConfig(env.config, { + database: env.database, ++ logger: env.logger, +}); +``` diff --git a/plugins/search-backend-module-pg/api-report.md b/plugins/search-backend-module-pg/api-report.md index d35bac66aa..3c6096090e 100644 --- a/plugins/search-backend-module-pg/api-report.md +++ b/plugins/search-backend-module-pg/api-report.md @@ -8,6 +8,7 @@ import { Config } from '@backstage/config'; import { IndexableDocument } from '@backstage/plugin-search-common'; import { IndexableResultSet } from '@backstage/plugin-search-common'; import { Knex } from 'knex'; +import { Logger } from 'winston'; import { PluginDatabaseManager } from '@backstage/backend-common'; import { SearchEngine } from '@backstage/plugin-search-common'; import { SearchQuery } from '@backstage/plugin-search-common'; @@ -84,11 +85,12 @@ export interface DocumentResultRow { // @public (undocumented) export class PgSearchEngine implements SearchEngine { // @deprecated - constructor(databaseStore: DatabaseStore, config: Config); + constructor(databaseStore: DatabaseStore, config: Config, logger?: Logger); // @deprecated (undocumented) static from(options: { database: PluginDatabaseManager; config: Config; + logger?: Logger; }): Promise; // (undocumented) static fromConfig( @@ -126,6 +128,7 @@ export type PgSearchEngineIndexerOptions = { batchSize: number; type: string; databaseStore: DatabaseStore; + logger?: Logger; }; // @public @@ -144,6 +147,7 @@ export type PgSearchHighlightOptions = { // @public export type PgSearchOptions = { database: PluginDatabaseManager; + logger?: Logger; }; // @public (undocumented) diff --git a/plugins/search-backend-module-pg/package.json b/plugins/search-backend-module-pg/package.json index 25ee3cf696..2e0f46f24a 100644 --- a/plugins/search-backend-module-pg/package.json +++ b/plugins/search-backend-module-pg/package.json @@ -29,7 +29,8 @@ "@backstage/plugin-search-common": "workspace:^", "knex": "^2.0.0", "lodash": "^4.17.21", - "uuid": "^8.3.2" + "uuid": "^8.3.2", + "winston": "^3.2.1" }, "devDependencies": { "@backstage/backend-test-utils": "workspace:^", diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index f53c433c2f..d2cf80e0ca 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -28,6 +28,7 @@ import { PgSearchQuery, } from '../database'; import { v4 as uuid } from 'uuid'; +import { Logger } from 'winston'; import { Config } from '@backstage/config'; /** @@ -62,6 +63,7 @@ export type PgSearchQueryTranslator = ( */ export type PgSearchOptions = { database: PluginDatabaseManager; + logger?: Logger; }; /** @@ -82,12 +84,17 @@ export type PgSearchHighlightOptions = { /** @public */ export class PgSearchEngine implements SearchEngine { + private readonly logger?: Logger; private readonly highlightOptions: PgSearchHighlightOptions; /** * @deprecated This will be marked as private in a future release, please us fromConfig instead */ - constructor(private readonly databaseStore: DatabaseStore, config: Config) { + constructor( + private readonly databaseStore: DatabaseStore, + config: Config, + logger?: Logger, + ) { const uuidTag = uuid(); const highlightConfig = config.getOptionalConfig( 'search.pg.highlightOptions', @@ -107,6 +114,7 @@ export class PgSearchEngine implements SearchEngine { highlightConfig?.getOptionalString('fragmentDelimiter') ?? ' ... ', }; this.highlightOptions = highlightOptions; + this.logger = logger; } /** @@ -115,10 +123,12 @@ export class PgSearchEngine implements SearchEngine { static async from(options: { database: PluginDatabaseManager; config: Config; + logger?: Logger; }): Promise { return new PgSearchEngine( await DatabaseDocumentStore.create(options.database), options.config, + options.logger, ); } @@ -126,6 +136,7 @@ export class PgSearchEngine implements SearchEngine { return new PgSearchEngine( await DatabaseDocumentStore.create(options.database), config, + options.logger, ); } @@ -170,6 +181,7 @@ export class PgSearchEngine implements SearchEngine { batchSize: 1000, type, databaseStore: this.databaseStore, + logger: this.logger?.child({ documentType: type }), }); } diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts index e3dbe82751..63dceec4dc 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts @@ -14,9 +14,11 @@ * limitations under the License. */ +import { getVoidLogger } from '@backstage/backend-common'; import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; import { IndexableDocument } from '@backstage/plugin-search-common'; import { Knex } from 'knex'; +import { Logger } from 'winston'; import { DatabaseStore } from '../database'; /** @public */ @@ -24,10 +26,12 @@ export type PgSearchEngineIndexerOptions = { batchSize: number; type: string; databaseStore: DatabaseStore; + logger?: Logger; }; /** @public */ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { + private logger: Logger; private store: DatabaseStore; private type: string; private tx: Knex.Transaction | undefined; @@ -36,6 +40,7 @@ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { super({ batchSize: options.batchSize }); this.store = options.databaseStore; this.type = options.type; + this.logger = options.logger || getVoidLogger(); } async initialize(): Promise { diff --git a/yarn.lock b/yarn.lock index e3d1b5eb88..c3a2a2a844 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7535,6 +7535,7 @@ __metadata: knex: ^2.0.0 lodash: ^4.17.21 uuid: ^8.3.2 + winston: ^3.2.1 languageName: unknown linkType: soft From 18646ccb1ad767d8c0225dbe50ddf5b510020373 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 29 Nov 2022 15:12:23 +0100 Subject: [PATCH 013/237] Update lunr engine to preserve pre-existing indices if indexer receives 0 documents Signed-off-by: Eric Peterson --- .../src/engines/LunrSearchEngine.test.ts | 56 ++++++++++++++++++- .../src/engines/LunrSearchEngine.ts | 29 +++++++++- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts index 4ef380943f..27cd65d63b 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts @@ -1026,7 +1026,7 @@ describe('LunrSearchEngine', () => { // Get the indexer and invoke its close handler. await inspectableSearchEngine.getIndexer('test-index'); - const onClose = indexerMock.on.mock.calls[0][1] as Function; + const onClose = indexerMock.on.mock.calls[1][1] as Function; onClose(); // Ensure mocked methods were called. @@ -1044,6 +1044,60 @@ describe('LunrSearchEngine', () => { 'new-location': doc, }); }); + + it('should not replace index or docs if no docs were indexed', async () => { + // Set up an inspectable search engine to pre-set some data. + const doc = { title: 'A doc', text: 'test', location: 'some-location' }; + const inspectableSearchEngine = new LunrSearchEngineForTests({ + logger: getVoidLogger(), + }); + inspectableSearchEngine.setDocStore({ 'existing-location': doc }); + + // Mock methods called by close handler (resolving no documents) + indexerMock.buildIndex.mockReturnValueOnce('expected-index'); + indexerMock.getDocumentStore.mockReturnValueOnce({}); + + // Get the indexer and invoke its close handler. + await inspectableSearchEngine.getIndexer('test-index'); + const onClose = indexerMock.on.mock.calls[1][1] as Function; + onClose(); + + // Ensure buildIndex method was not called. + expect(indexerMock.buildIndex).not.toHaveBeenCalled(); + + // Ensure pre-existing documents still exist in the store + expect(inspectableSearchEngine.getDocStore()).toStrictEqual({ + 'existing-location': doc, + }); + }); + + it('should not replace index or docs if an error was thrown', async () => { + // Set up an inspectable search engine to pre-set some data. + const doc = { title: 'A doc', text: 'test', location: 'some-location' }; + const inspectableSearchEngine = new LunrSearchEngineForTests({ + logger: getVoidLogger(), + }); + inspectableSearchEngine.setDocStore({ 'existing-location': doc }); + + // Mock methods called by close handler (resolving no documents) + indexerMock.buildIndex.mockReturnValueOnce('expected-index'); + indexerMock.getDocumentStore.mockReturnValueOnce({}); + + // Get the indexer and invoke its close handler after firing an error. + await inspectableSearchEngine.getIndexer('test-index'); + const onError = indexerMock.on.mock.calls[0][1] as Function; + const onClose = indexerMock.on.mock.calls[1][1] as Function; + onError(new Error('Some collator error')); + onClose(); + + // Ensure buildIndex method was not called. + expect(indexerMock.buildIndex).not.toHaveBeenCalled(); + + // Ensure pre-existing documents still exist in the store + expect(inspectableSearchEngine.getDocStore()).toStrictEqual({ + 'existing-location': doc, + }); + }); }); }); diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts index ff2c828402..b69b5ea863 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts @@ -153,12 +153,37 @@ export class LunrSearchEngine implements SearchEngine { async getIndexer(type: string) { const indexer = new LunrSearchEngineIndexer(); + const indexerLogger = this.logger.child({ documentType: type }); + let errorThrown: Error | undefined; + + indexer.on('error', err => { + errorThrown = err; + }); indexer.on('close', () => { // Once the stream is closed, build the index and store the documents in // memory for later retrieval. - this.lunrIndices[type] = indexer.buildIndex(); - this.docStore = { ...this.docStore, ...indexer.getDocumentStore() }; + const newDocuments = indexer.getDocumentStore(); + const docStoreExists = this.lunrIndices[type] !== undefined; + const documentsIndexed = Object.keys(newDocuments).length; + + // Do not set the index if there was an error or if no documents were + // indexed. This ensures search continues to work for an index, even in + // case of transient issues in underlying collators. + if (!errorThrown && documentsIndexed > 0) { + this.lunrIndices[type] = indexer.buildIndex(); + this.docStore = { ...this.docStore, ...newDocuments }; + } else { + indexerLogger.warn( + `Index for ${type} was not ${ + docStoreExists ? 'replaced' : 'created' + }: ${ + errorThrown + ? 'an error was encountered' + : 'indexer received 0 documents' + }`, + ); + } }); return indexer; From b24cf5a99e4ae027169b4e727947ebb367250f1b Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 29 Nov 2022 15:13:31 +0100 Subject: [PATCH 014/237] Update pg engine to preserve pre-existing indices if indexer receives 0 documents Signed-off-by: Eric Peterson --- .../PgSearchEngine/PgSearchEngineIndexer.test.ts | 9 +++++++++ .../src/PgSearchEngine/PgSearchEngineIndexer.ts | 14 ++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts index 0b9cd96c19..f1cb3923e1 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts @@ -81,6 +81,15 @@ describe('PgSearchEngineIndexer', () => { expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); }); + it('should rollback transaction if no documents indexed', async () => { + await TestPipeline.fromIndexer(indexer).withDocuments([]).execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).not.toHaveBeenCalled(); + expect(database.completeInsert).not.toHaveBeenCalled(); + expect(tx.rollback).toHaveBeenCalled(); + }); + it('should close out stream and bubble up error on prepare', async () => { const expectedError = new Error('Prepare error'); const documents = [ diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts index 63dceec4dc..ba625bc116 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts @@ -35,6 +35,7 @@ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { private store: DatabaseStore; private type: string; private tx: Knex.Transaction | undefined; + private numRecords = 0; constructor(options: PgSearchEngineIndexerOptions) { super({ batchSize: options.batchSize }); @@ -56,6 +57,8 @@ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { } async index(documents: IndexableDocument[]): Promise { + this.numRecords += documents.length; + try { await this.store.insertDocuments(this.tx!, this.type, documents); } catch (e) { @@ -67,6 +70,17 @@ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { } async finalize(): Promise { + // If no documents were indexed, rollback the transaction, log a warning, + // and do not continue. This ensures that collators that return empty sets + // of documents do not cause the index to be deleted. + if (this.numRecords === 0) { + this.logger.warn( + `Index for ${this.type} was not replaced: indexer received 0 documents`, + ); + this.tx!.rollback!(); + return; + } + // Attempt to complete and commit the transaction. try { await this.store.completeInsert(this.tx!, this.type); From 35af08d9a03afea0584a951e9d4395ff462315e0 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 29 Nov 2022 15:14:30 +0100 Subject: [PATCH 015/237] Update elastic engine to preserve pre-existing indices if indexer receives 0 documents Signed-off-by: Eric Peterson --- .../src/engines/ElasticSearchSearchEngine.ts | 11 ++++++---- .../ElasticSearchSearchEngineIndexer.test.ts | 20 +++++++++++++++++++ .../ElasticSearchSearchEngineIndexer.ts | 18 +++++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index f3db46956d..f17ec1b01d 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -252,6 +252,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { async getIndexer(type: string) { const alias = this.constructSearchAlias(type); + const indexerLogger = this.logger.child({ documentType: type }); const indexer = new ElasticSearchSearchEngineIndexer({ type, @@ -259,13 +260,13 @@ export class ElasticSearchSearchEngine implements SearchEngine { indexSeparator: this.indexSeparator, alias, elasticSearchClientWrapper: this.elasticSearchClientWrapper, - logger: this.logger, + logger: indexerLogger, batchSize: this.batchSize, }); // Attempt cleanup upon failure. indexer.on('error', async e => { - this.logger.error(`Failed to index documents for type ${type}`, e); + indexerLogger.error(`Failed to index documents for type ${type}`, e); let cleanupError: Error | undefined; // In some cases, a failure may have occurred before the indexer was able @@ -296,11 +297,13 @@ export class ElasticSearchSearchEngine implements SearchEngine { }); if (cleanupError) { - this.logger.error( + indexerLogger.error( `Unable to clean up elastic index ${indexer.indexName}: ${cleanupError}`, ); } else { - this.logger.info(`Removed partial, failed index ${indexer.indexName}`); + indexerLogger.info( + `Removed partial, failed index ${indexer.indexName}`, + ); } }); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts index 155a6bbf32..1e9e98dbaf 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -176,6 +176,26 @@ describe('ElasticSearchSearchEngineIndexer', () => { expect(deleteSpy).toHaveBeenCalled(); }); + it('handles when no documents are received', async () => { + await TestPipeline.fromIndexer(indexer).withDocuments([]).execute(); + + // Older indices should have been queried for. + expect(catSpy).toHaveBeenCalled(); + + // A new index should have been created. + const createdIndex = createSpy.mock.calls[0][0].path.slice(1); + expect(createdIndex).toContain('some-type-index__'); + + // No documents should have been sent + expect(bulkSpy).not.toHaveBeenCalled(); + + // Alias should not have been rotated. + expect(aliasesSpy).not.toHaveBeenCalled(); + + // Old index should not be cleaned up. + expect(deleteSpy).not.toHaveBeenCalled(); + }); + it('handles bulk and batching during indexing', async () => { const documents = range(550).map(i => ({ title: `Hello World ${i}`, diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index e32c49559d..1b1d47bb9b 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -131,6 +131,24 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { // Wait for the bulk helper to finish processing. const result = await this.bulkResult; + // Warn that no documents were indexed, early return so that alias swapping + // does not occur, and clean up the empty index we just created. + if (this.processed === 0) { + this.logger.warn( + `Index for ${this.type} was not ${ + this.removableIndices.length ? 'replaced' : 'created' + }: indexer received 0 documents`, + ); + try { + await this.elasticSearchClientWrapper.deleteIndex({ + index: this.indexName, + }); + } catch (error) { + this.logger.error(`Unable to clean up elastic index: ${error}`); + } + return; + } + // Rotate main alias upon completion. Apply permanent secondary alias so // stale indices can be referenced for deletion in case initial attempt // fails. Allow errors to bubble up so that we can clean up the created index. From dff98437181c229b20f87f3c457d89ab37d22db5 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 29 Nov 2022 15:20:10 +0100 Subject: [PATCH 016/237] Changeset describing updated search indexer behavior Signed-off-by: Eric Peterson --- .changeset/search-with-alcohol.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/search-with-alcohol.md diff --git a/.changeset/search-with-alcohol.md b/.changeset/search-with-alcohol.md new file mode 100644 index 0000000000..a46f33985a --- /dev/null +++ b/.changeset/search-with-alcohol.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': minor +'@backstage/plugin-search-backend-module-pg': minor +'@backstage/plugin-search-backend-node': minor +--- + +The search engine now better handles the case when it receives 0 documents at index-time. Prior to this change, the indexer would replace any existing index with an empty index, effectively deleting it. Now instead, a warning is logged, and any existing index is left alone (preserving the index from the last successful indexing attempt). From d75d73d33bfb2d21f8e8cea10d35c98dc39fea50 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 2 Dec 2022 11:46:47 +0100 Subject: [PATCH 017/237] Clearer test assertions Co-authored-by: Renan Mendes Carvalho Signed-off-by: Eric Peterson --- .../src/engines/ElasticSearchSearchEngineIndexer.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts index 1e9e98dbaf..92193976b4 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -183,8 +183,12 @@ describe('ElasticSearchSearchEngineIndexer', () => { expect(catSpy).toHaveBeenCalled(); // A new index should have been created. - const createdIndex = createSpy.mock.calls[0][0].path.slice(1); - expect(createdIndex).toContain('some-type-index__'); + expect(createSpy).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + path: expect.stringContaining('some-type-index__'), + }), + ); // No documents should have been sent expect(bulkSpy).not.toHaveBeenCalled(); From b5e3da44c4155fd2b074696a05dc8c2a492f668b Mon Sep 17 00:00:00 2001 From: TheMonolithX64 Date: Fri, 2 Dec 2022 17:42:20 +0000 Subject: [PATCH 018/237] Update broken links to cli docs Signed-off-by: TheMonolithX64 --- docs/getting-started/keeping-backstage-updated.md | 4 ++-- packages/backend/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/getting-started/keeping-backstage-updated.md b/docs/getting-started/keeping-backstage-updated.md index 64f78c0997..8e130e7094 100644 --- a/docs/getting-started/keeping-backstage-updated.md +++ b/docs/getting-started/keeping-backstage-updated.md @@ -13,7 +13,7 @@ starting point that's meant to be evolved. The Backstage CLI has a command to bump all `@backstage` packages and dependencies you're using to the latest versions: -[versions:bump](https://backstage.io/docs/cli/commands#versionsbump). +[versions:bump](https://backstage.io/docs/local-dev/cli-commands#versionsbump). ```bash yarn backstage-cli versions:bump @@ -70,7 +70,7 @@ example, depends on global referential equality. This can cause problems in Backstage with API lookup, or config loading. To help resolve these situations, the Backstage CLI has -[versions:check](https://backstage.io/docs/cli/commands#versionscheck). This +[versions:check](https://backstage.io/docs/local-dev/cli-commands#versionscheck). This will validate versions of `@backstage` packages in your app to check for duplicate definitions: diff --git a/packages/backend/README.md b/packages/backend/README.md index f4c115db65..e4fd81749d 100644 --- a/packages/backend/README.md +++ b/packages/backend/README.md @@ -31,7 +31,7 @@ The backend starts up on port 7007 per default. ### Debugging -The backend is a node process that can be inspected to allow breakpoints and live debugging. To enable this, pass the `--inspect` flag to [backend:dev](https://backstage.io/docs/cli/commands#backenddev). +The backend is a node process that can be inspected to allow breakpoints and live debugging. To enable this, pass the `--inspect` flag to [backend:dev](https://backstage.io/docs/local-dev/cli-build-system#backend-development). To debug the backend in [Visual Studio Code](https://code.visualstudio.com/): From facbae6feff8b6715b75722e499f4fe876a8d6f7 Mon Sep 17 00:00:00 2001 From: iris Date: Thu, 1 Dec 2022 17:00:32 +0800 Subject: [PATCH 019/237] Add last_updated_at in final_entities table Signed-off-by: iris --- ...5_add_last_updated_at_in_final_entities.js | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js diff --git a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js new file mode 100644 index 0000000000..9afd32b774 --- /dev/null +++ b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js @@ -0,0 +1,31 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +exports.up = async function up(knex) { + await knex.schema.table('final_entities', table => { + table.timestamp('last_updated_at').nullable(); + }); +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function down(knex) { + await knex.schema.table('final_entities', table => { + table.dropColumn('last_updated_at'); + }); +}; From 253acc8b4395b2bdac6afdf651aba3b8c15bdc5f Mon Sep 17 00:00:00 2001 From: iris Date: Thu, 1 Dec 2022 17:08:36 +0800 Subject: [PATCH 020/237] update last_updated_at when update final_entities Signed-off-by: iris --- plugins/catalog-backend/src/database/tables.ts | 1 + plugins/catalog-backend/src/stitching/Stitcher.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index c23753b267..e3f3efd1bb 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -66,6 +66,7 @@ export type DbFinalEntitiesRow = { hash: string; stitch_ticket: string; final_entity?: string; + last_updated_at?: string | Date; }; export type DbSearchRow = { diff --git a/plugins/catalog-backend/src/stitching/Stitcher.ts b/plugins/catalog-backend/src/stitching/Stitcher.ts index 0da8e39694..64f5527262 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.ts @@ -207,6 +207,7 @@ export class Stitcher { .update({ final_entity: JSON.stringify(entity), hash, + last_updated_at: this.database.fn.now(), }) .where('entity_id', entityId) .where('stitch_ticket', ticket) From a483d5f5a4d82219e4e41e764127020de02ea9c9 Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 2 Dec 2022 12:05:18 +0800 Subject: [PATCH 021/237] add last_updated_at as annotation backstage.io/last_updated-at Signed-off-by: iris --- .../src/service/DefaultEntitiesCatalog.ts | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 14f22f862b..1a0fe98b40 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -210,7 +210,14 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { }; } - let entities: Entity[] = rows.map(e => JSON.parse(e.final_entity!)); + let entities: Entity[] = rows.map(e => { + const entityJson = JSON.parse(e.final_entity!); + if (e.last_updated_at) { + entityJson.metadata.annotations['backstage.io/last_updated-at'] = + e.last_updated_at; + } + return entityJson; + }); if (request?.fields) { entities = entities.map(e => request.fields!(e)); @@ -263,7 +270,12 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { query = parseFilter(request.filter, query, this.database); } for (const row of await query) { - lookup.set(row.entityRef, row.entity ? JSON.parse(row.entity) : null); + const entityJson = JSON.parse(row.entity); + if (row.entity.last_updated_at) { + entityJson.metadata.annotations['backstage.io/last_updated-at'] = + row.entity.last_updated_at; + } + lookup.set(row.entityRef, row.entity ? entityJson : null); } } @@ -373,13 +385,18 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { .where('refresh_state.entity_ref', '=', rootRef) .select({ entityJson: 'final_entities.final_entity', + last_updated_at: 'final_entities.last_updated_at', }); if (!rootRow) { throw new NotFoundError(`No such entity ${rootRef}`); } - - const rootEntity = JSON.parse(rootRow.entityJson) as Entity; + const entityJson = JSON.parse(rootRow.entityJson); + if (rootRow.last_updated_at) { + entityJson.metadata.annotations['backstage.io/last_updated-at'] = + rootRow.last_updated_at; + } + const rootEntity = entityJson as Entity; const seenEntityRefs = new Set(); const todo = new Array(); const items = new Array<{ entity: Entity; parentEntityRefs: string[] }>(); From 93870e4df1a1a7d9f5f09fed5c333bcf5f97735f Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 2 Dec 2022 13:00:31 +0800 Subject: [PATCH 022/237] Add changeset Signed-off-by: iris --- .changeset/lucky-singers-worry.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/lucky-singers-worry.md diff --git a/.changeset/lucky-singers-worry.md b/.changeset/lucky-singers-worry.md new file mode 100644 index 0000000000..742883e784 --- /dev/null +++ b/.changeset/lucky-singers-worry.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': major +--- + +Track the last time the final entity changed with new timestamp "last_updated_at" data in final_entities database, which gets updated with the time when final_entities is updated. And it's displayed in metadata annotation as backstage.io/last_updated-at. From 06f6a4f0f14de66fe5372e5f9d773e9076ee3e96 Mon Sep 17 00:00:00 2001 From: Scott Guymer Date: Fri, 2 Dec 2022 16:48:22 +0100 Subject: [PATCH 023/237] fix: allow overriding of stackoverflow configuration Make it possible to override the config when instantiating for more flexibility. Signed-off-by: Scott Guymer --- .changeset/olive-eyes-sing.md | 5 +++++ .../src/search/StackOverflowQuestionsCollatorFactory.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/olive-eyes-sing.md diff --git a/.changeset/olive-eyes-sing.md b/.changeset/olive-eyes-sing.md new file mode 100644 index 0000000000..d0ae1dc08f --- /dev/null +++ b/.changeset/olive-eyes-sing.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-stack-overflow-backend': minor +--- + +Enable configuration override for StackOverflow backend plugin when instantiating the search indexer. This makes it possible to set different configuration for frontend and backend of the plugin. diff --git a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts index ebbad99d0a..807d6637c1 100644 --- a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts +++ b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts @@ -95,11 +95,11 @@ export class StackOverflowQuestionsCollatorFactory 'https://api.stackexchange.com/2.2'; const maxPage = options.maxPage || 100; return new StackOverflowQuestionsCollatorFactory({ - ...options, baseUrl, maxPage, apiKey, apiAccessToken, + ...options, }); } From 6151f8e07166a25e7940a303609ab11bdd91adcc Mon Sep 17 00:00:00 2001 From: Scott Guymer Date: Mon, 5 Dec 2022 09:19:49 +0100 Subject: [PATCH 024/237] Added some basic tests for stackoverflow backend plugin Signed-off-by: Scott Guymer --- plugins/stack-overflow-backend/package.json | 8 +- ...ckOverflowQuestionsCollatorFactory.test.ts | 151 ++++++++++++++++++ 2 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts diff --git a/plugins/stack-overflow-backend/package.json b/plugins/stack-overflow-backend/package.json index 5a58f54958..1750ddbc6c 100644 --- a/plugins/stack-overflow-backend/package.json +++ b/plugins/stack-overflow-backend/package.json @@ -32,13 +32,19 @@ "clean": "backstage-cli package clean" }, "dependencies": { - "@backstage/cli": "workspace:^", + "@backstage/backend-common": "workspace:^", "@backstage/config": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "node-fetch": "^2.6.7", "qs": "^6.9.4", "winston": "^3.2.1" }, + "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", + "@backstage/cli": "workspace:^", + "@backstage/plugin-search-backend-node": "workspace:^", + "msw": "^0.49.0" + }, "files": [ "dist", "config.d.ts" diff --git a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts new file mode 100644 index 0000000000..9f2c316be4 --- /dev/null +++ b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { getVoidLogger } from '@backstage/backend-common'; +import { + StackOverflowQuestionsCollatorFactory, + StackOverflowQuestionsCollatorFactoryOptions, +} from './StackOverflowQuestionsCollatorFactory'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { ConfigReader } from '@backstage/config'; +import { Readable } from 'stream'; +import { setupServer } from 'msw/node'; +import { rest } from 'msw'; + +const logger = getVoidLogger(); + +const mockQuestion = { + items: [ + { + tags: ['backstage'], + owner: { + display_name: 'The Riddler', + }, + answer_count: 1, + link: 'https://stack.overflow.local/questions/2911', + title: 'This is the first question', + }, + ], + has_more: false, +}; + +const mockOverrideQuestion = { + items: [ + { + tags: ['backstage'], + owner: { + display_name: 'The Riddler', + }, + answer_count: 1, + link: 'https://stack.overflow.local/questions/1', + title: 'This is the first question', + }, + { + tags: ['backstage'], + owner: { + display_name: 'The Riddler', + }, + answer_count: 1, + link: 'https://stack.overflow.local/questions/2', + title: 'this is another question', + }, + ], + has_more: false, +}; + +describe('StackOverflowQuestionsCollatorFactory', () => { + const config = new ConfigReader({ + stackoverflow: { + baseUrl: 'http://stack.overflow.local', + }, + }); + + const defaultOptions: StackOverflowQuestionsCollatorFactoryOptions = { + logger, + requestParams: { + tagged: ['developer-portal'], + pagesize: 100, + order: 'desc', + sort: 'activity', + }, + }; + + it('has expected type', () => { + const factory = StackOverflowQuestionsCollatorFactory.fromConfig( + config, + defaultOptions, + ); + expect(factory.type).toBe('stack-overflow'); + }); + + describe('getCollator', () => { + const worker = setupServer(); + setupRequestMockHandlers(worker); + + afterEach(async () => { + worker.resetHandlers(); + }); + + afterAll(async () => { + worker.close(); + }); + + it('returns a readable stream', async () => { + const factory = StackOverflowQuestionsCollatorFactory.fromConfig( + config, + defaultOptions, + ); + const collator = await factory.getCollator(); + expect(collator).toBeInstanceOf(Readable); + }); + + it('fetches from the configured endpoint', async () => { + worker.use( + rest.get('http://stack.overflow.local/questions', (_, res, ctx) => + res(ctx.status(200), ctx.json(mockQuestion)), + ), + ); + const factory = StackOverflowQuestionsCollatorFactory.fromConfig( + config, + defaultOptions, + ); + const collator = await factory.getCollator(); + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + + expect(documents).toHaveLength(mockQuestion.items.length); + }); + + it('fetches from the overridden endpoint', async () => { + worker.use( + rest.get('http://stack.overflow.override/questions', (_, res, ctx) => + res(ctx.status(200), ctx.json(mockOverrideQuestion)), + ), + ); + const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { + logger, + baseUrl: 'http://stack.overflow.override', + requestParams: defaultOptions.requestParams, + }); + const collator = await factory.getCollator(); + + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + + expect(documents).toHaveLength(mockOverrideQuestion.items.length); + }); + }); +}); From 9e4725f8f4736dbccccef47a8ed640abc383e5c0 Mon Sep 17 00:00:00 2001 From: Scott Guymer Date: Mon, 5 Dec 2022 09:38:20 +0100 Subject: [PATCH 025/237] Updated yarn.lock Signed-off-by: Scott Guymer --- yarn.lock | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/yarn.lock b/yarn.lock index 4af9e1862a..9b8f80db7e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7845,9 +7845,13 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/plugin-stack-overflow-backend@workspace:plugins/stack-overflow-backend" dependencies: + "@backstage/backend-common": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" + msw: ^0.49.0 node-fetch: ^2.6.7 qs: ^6.9.4 winston: ^3.2.1 From 68cf4f593416e71248c4e38f9654bfccc351f83c Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 2 Dec 2022 13:02:43 +0800 Subject: [PATCH 026/237] fix testing Signed-off-by: iris --- .changeset/lucky-singers-worry.md | 2 +- ...5_add_last_updated_at_in_final_entities.js | 5 ++- .../catalog-backend/src/database/tables.ts | 2 +- .../src/service/DefaultEntitiesCatalog.ts | 6 ++-- .../src/stitching/Stitcher.test.ts | 35 +++++++++++++++++-- .../catalog-backend/src/stitching/Stitcher.ts | 3 +- 6 files changed, 44 insertions(+), 9 deletions(-) diff --git a/.changeset/lucky-singers-worry.md b/.changeset/lucky-singers-worry.md index 742883e784..abe581fbc3 100644 --- a/.changeset/lucky-singers-worry.md +++ b/.changeset/lucky-singers-worry.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend': major --- -Track the last time the final entity changed with new timestamp "last_updated_at" data in final_entities database, which gets updated with the time when final_entities is updated. And it's displayed in metadata annotation as backstage.io/last_updated-at. +Track the last time the final entity changed with new timestamp "last updated at" data in final entities database, which gets updated with the time when final entity is updated. And it's displayed in metadata annotation. diff --git a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js index 9afd32b774..d12d5bec3e 100644 --- a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js +++ b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js @@ -16,7 +16,10 @@ exports.up = async function up(knex) { await knex.schema.table('final_entities', table => { - table.timestamp('last_updated_at').nullable(); + table + .bigint('last_updated_at') + .nullable() + .comment('The time when final_entity changed'); }); }; diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index e3f3efd1bb..2778cf2fba 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -66,7 +66,7 @@ export type DbFinalEntitiesRow = { hash: string; stitch_ticket: string; final_entity?: string; - last_updated_at?: string | Date; + last_updated_at: string | null; }; export type DbSearchRow = { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 1a0fe98b40..a305df52e3 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -213,7 +213,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { let entities: Entity[] = rows.map(e => { const entityJson = JSON.parse(e.final_entity!); if (e.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated-at'] = + entityJson.metadata.annotations['backstage.io/last_updated_at'] = e.last_updated_at; } return entityJson; @@ -272,7 +272,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { for (const row of await query) { const entityJson = JSON.parse(row.entity); if (row.entity.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated-at'] = + entityJson.metadata.annotations['backstage.io/last_updated_at'] = row.entity.last_updated_at; } lookup.set(row.entityRef, row.entity ? entityJson : null); @@ -393,7 +393,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } const entityJson = JSON.parse(rootRow.entityJson); if (rootRow.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated-at'] = + entityJson.metadata.annotations['backstage.io/last_updated_at'] = rootRow.last_updated_at; } const rootEntity = entityJson as Entity; diff --git a/plugins/catalog-backend/src/stitching/Stitcher.test.ts b/plugins/catalog-backend/src/stitching/Stitcher.test.ts index a3657e45fd..a13467e9bd 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.test.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.test.ts @@ -17,6 +17,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { TestDatabases } from '@backstage/backend-test-utils'; import { Entity } from '@backstage/catalog-model'; +import { DateTime } from 'luxon'; import { applyDatabaseMigrations } from '../database/migrations'; import { DbFinalEntitiesRow, @@ -42,6 +43,7 @@ describe('Stitcher', () => { const stitcher = new Stitcher(db, logger); let entities: DbFinalEntitiesRow[]; let entity: Entity; + const timeBeforeStitch = DateTime.now(); await db('refresh_state').insert([ { @@ -110,6 +112,15 @@ describe('Stitcher', () => { }); expect(entity.metadata.etag).toEqual(entities[0].hash); + const last_updated_at = entities[0].last_updated_at; + expect(last_updated_at).not.toBeNull(); + const lastUpdatedAt = DateTime.fromMillis( + last_updated_at ? +last_updated_at : 0, + ); + const msAfterStitch = lastUpdatedAt + .diff(timeBeforeStitch, 'milliseconds') + .toObject(); + expect(msAfterStitch.milliseconds).toBeGreaterThan(0); const firstHash = entities[0].hash; const search = await db('search'); @@ -127,7 +138,12 @@ describe('Stitcher', () => { original_value: 'a', value: 'a', }, - { entity_id: 'my-id', key: 'kind', original_value: 'k', value: 'k' }, + { + entity_id: 'my-id', + key: 'kind', + original_value: 'k', + value: 'k', + }, { entity_id: 'my-id', key: 'metadata.name', @@ -174,6 +190,7 @@ describe('Stitcher', () => { }, ]); + const timeBeforeRestitch = DateTime.now(); await stitcher.stitch(new Set(['k:ns/n'])); entities = await db('final_entities'); @@ -206,6 +223,15 @@ describe('Stitcher', () => { expect(entities[0].hash).not.toEqual(firstHash); expect(entities[0].hash).toEqual(entity.metadata.etag); + expect(entity.metadata.etag).toEqual(entities[0].hash); + const last_updated_at_after_restitch = entities[0].last_updated_at; + expect(last_updated_at_after_restitch).not.toBeNull(); + const msAfterRestitch = DateTime.fromMillis( + last_updated_at_after_restitch ? +last_updated_at_after_restitch : 0, + ) + .diff(timeBeforeRestitch, 'milliseconds') + .toObject(); + expect(msAfterRestitch.milliseconds).toBeGreaterThan(0); expect(await db('search')).toEqual( expect.arrayContaining([ @@ -227,7 +253,12 @@ describe('Stitcher', () => { original_value: 'a', value: 'a', }, - { entity_id: 'my-id', key: 'kind', original_value: 'k', value: 'k' }, + { + entity_id: 'my-id', + key: 'kind', + original_value: 'k', + value: 'k', + }, { entity_id: 'my-id', key: 'metadata.name', diff --git a/plugins/catalog-backend/src/stitching/Stitcher.ts b/plugins/catalog-backend/src/stitching/Stitcher.ts index 64f5527262..c3892244dd 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.ts @@ -23,6 +23,7 @@ import { import { SerializedError, stringifyError } from '@backstage/errors'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; +import { DateTime } from 'luxon'; import { Logger } from 'winston'; import { DbFinalEntitiesRow, @@ -207,7 +208,7 @@ export class Stitcher { .update({ final_entity: JSON.stringify(entity), hash, - last_updated_at: this.database.fn.now(), + last_updated_at: `${DateTime.now().toMillis()}`, }) .where('entity_id', entityId) .where('stitch_ticket', ticket) From 2f52b1274009c3542f578d290a894885f5d1b75e Mon Sep 17 00:00:00 2001 From: Scott Guymer Date: Wed, 7 Dec 2022 17:13:11 +0100 Subject: [PATCH 027/237] Remove cleanup calls as they are handled in the setup of msw Signed-off-by: Scott Guymer --- .../search/StackOverflowQuestionsCollatorFactory.test.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts index 9f2c316be4..bdc5be125c 100644 --- a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts +++ b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts @@ -95,14 +95,6 @@ describe('StackOverflowQuestionsCollatorFactory', () => { const worker = setupServer(); setupRequestMockHandlers(worker); - afterEach(async () => { - worker.resetHandlers(); - }); - - afterAll(async () => { - worker.close(); - }); - it('returns a readable stream', async () => { const factory = StackOverflowQuestionsCollatorFactory.fromConfig( config, From 833872e55b758ea57d0212f66738b5009ba90973 Mon Sep 17 00:00:00 2001 From: djamaile Date: Fri, 9 Dec 2022 11:19:02 +0100 Subject: [PATCH 028/237] chore: use changeset feedback action Signed-off-by: djamaile --- .../workflows/automate_changeset_feedback.yml | 73 +------------------ 1 file changed, 4 insertions(+), 69 deletions(-) diff --git a/.github/workflows/automate_changeset_feedback.yml b/.github/workflows/automate_changeset_feedback.yml index 700eae7d0c..b6b5ea3d7e 100644 --- a/.github/workflows/automate_changeset_feedback.yml +++ b/.github/workflows/automate_changeset_feedback.yml @@ -21,75 +21,10 @@ jobs: if: github.repository == 'backstage/backstage' && github.event.pull_request.user.login != 'backstage-service' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: backstage/actions/changeset-feedback@v0.5.9 with: # Fetch the commit that's merged into the base rather than the target ref # This will let us diff only the contents of the PR, without fetching more history - ref: 'refs/pull/${{ github.event.pull_request.number }}/merge' - - - name: fetch base - run: git fetch --depth 1 origin ${{ github.base_ref }} - - # We avoid using the in-source script since this workflow has elevated permissions that we don't want to expose - - name: Generate Feedback - id: generate-feedback - run: | - rm -f generate.js - wget -O generate.js https://raw.githubusercontent.com/backstage/backstage/master/scripts/generate-changeset-feedback.js 1>&2 - node generate.js FETCH_HEAD > feedback.txt - - - name: Post Feedback - uses: actions/github-script@v6 - env: - ISSUE_NUMBER: ${{ github.event.pull_request.number }} - with: - script: | - const owner = "backstage"; - const repo = "backstage"; - const marker = ""; - const feedback = require('fs').readFileSync('feedback.txt', 'utf8'); - const issue_number = Number(process.env.ISSUE_NUMBER); - const body = feedback.trim() ? feedback + marker : undefined - - const existingComments = await github.paginate(github.rest.issues.listComments, { - owner, - repo, - issue_number, - }); - - const existingComment = existingComments.find((c) => - c.user.login === "github-actions[bot]" && - c.body.includes(marker) - ); - - if (existingComment) { - if (body) { - if (existingComment.body !== body) { - console.log(`updating existing comment in #${issue_number}`); - await github.rest.issues.updateComment({ - owner, - repo, - comment_id: existingComment.id, - body, - }); - } else { - console.log(`skipped update of identical comment in #${issue_number}`); - } - } else { - console.log(`removing comment from #${issue_number}`); - await github.rest.issues.deleteComment({ - owner, - repo, - comment_id: existingComment.id, - body, - }); - } - } else if (body) { - console.log(`creating comment for #${issue_number}`); - await github.rest.issues.createComment({ - owner, - repo, - issue_number, - body, - }); - } + diffRef: 'refs/pull/${{ github.event.pull_request.number }}/merge' + github-token: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }} + issue-number: ${{ steps.pr-number.outputs.pr-number }} From a162f33a5f9ca17071b9418b14ed3e36dbfadbcb Mon Sep 17 00:00:00 2001 From: Djam Date: Fri, 9 Dec 2022 15:32:02 +0100 Subject: [PATCH 029/237] Update automate_changeset_feedback.yml Signed-off-by: Djam --- .github/workflows/automate_changeset_feedback.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/automate_changeset_feedback.yml b/.github/workflows/automate_changeset_feedback.yml index b6b5ea3d7e..3637ed11e7 100644 --- a/.github/workflows/automate_changeset_feedback.yml +++ b/.github/workflows/automate_changeset_feedback.yml @@ -21,10 +21,13 @@ jobs: if: github.repository == 'backstage/backstage' && github.event.pull_request.user.login != 'backstage-service' runs-on: ubuntu-latest steps: - - uses: backstage/actions/changeset-feedback@v0.5.9 + - uses: backstage/actions/changeset-feedback@vchangeset-patches with: # Fetch the commit that's merged into the base rather than the target ref # This will let us diff only the contents of the PR, without fetching more history diffRef: 'refs/pull/${{ github.event.pull_request.number }}/merge' github-token: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }} issue-number: ${{ steps.pr-number.outputs.pr-number }} + app-id: ${{ secrets.BACKSTAGE_GOALIE_APPLICATION_ID }} + private-key: ${{ secrets.BACKSTAGE_GOALIE_PRIVATE_KEY }} + installation-id: ${{ secrets.BACKSTAGE_GOALIE_INSTALLATION_ID }} From 273ba3a77fd246a3a10ab150d8a35dadc5fdea62 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 9 Dec 2022 16:25:44 +0100 Subject: [PATCH 030/237] catalog: register OpenTelemetry metrics, deprecate prom-client Signed-off-by: Johan Haals --- .changeset/healthy-waves-compare.md | 5 + plugins/catalog-backend/package.json | 1 + .../DefaultCatalogProcessingEngine.ts | 99 +++++++++++++++---- yarn.lock | 70 ++++++++++++- 4 files changed, 152 insertions(+), 23 deletions(-) create mode 100644 .changeset/healthy-waves-compare.md diff --git a/.changeset/healthy-waves-compare.md b/.changeset/healthy-waves-compare.md new file mode 100644 index 0000000000..cdd19d1450 --- /dev/null +++ b/.changeset/healthy-waves-compare.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Deprecated Prometheus metrics in favour of OpenTelemtry metrics. diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index fe8ccc609d..e29c1d3c9a 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -47,6 +47,7 @@ "@backstage/plugin-scaffolder-common": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@backstage/types": "workspace:^", + "@opentelemetry/api": "^1.3.0", "@types/express": "^4.17.6", "codeowners-utils": "^1.0.2", "core-js": "^3.6.5", diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index bef2878099..a27e5a0ce4 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -23,6 +23,7 @@ import { assertError, serializeError, stringifyError } from '@backstage/errors'; import { Hash } from 'crypto'; import stableStringify from 'fast-json-stable-stringify'; import { Logger } from 'winston'; +import { metrics } from '@opentelemetry/api'; import { ProcessingDatabase, RefreshStateItem } from '../database/types'; import { createCounterMetric, createSummaryMetric } from '../util/metrics'; import { @@ -257,62 +258,124 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { // Helps wrap the timing and logging behaviors function progressTracker() { - const stitchedEntities = createCounterMetric({ + // prom-client metrics are deprecated in favour of OpenTelemetry metrics. + const promStitchedEntities = createCounterMetric({ name: 'catalog_stitched_entities_count', - help: 'Amount of entities stitched', + help: 'Amount of entities stitched. DEPRECATED, use OpenTelemetry metrics instead', }); - const processedEntities = createCounterMetric({ + const promProcessedEntities = createCounterMetric({ name: 'catalog_processed_entities_count', - help: 'Amount of entities processed', + help: 'Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead', labelNames: ['result'], }); - const processingDuration = createSummaryMetric({ + const promProcessingDuration = createSummaryMetric({ name: 'catalog_processing_duration_seconds', - help: 'Time spent executing the full processing flow', + help: 'Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead', labelNames: ['result'], }); - const processorsDuration = createSummaryMetric({ + const promProcessorsDuration = createSummaryMetric({ name: 'catalog_processors_duration_seconds', - help: 'Time spent executing catalog processors', + help: 'Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead', labelNames: ['result'], }); - const processingQueueDelay = createSummaryMetric({ + const promProcessingQueueDelay = createSummaryMetric({ name: 'catalog_processing_queue_delay_seconds', - help: 'The amount of delay between being scheduled for processing, and the start of actually being processed', + help: 'The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead', }); + const meter = metrics.getMeter('default'); + const stitchedEntities = meter.createCounter( + 'catalog.stitched.entities.count', + { + description: 'Amount of entities stitched', + }, + ); + + const processedEntities = meter.createCounter( + 'catalog.stitched.entities.count', + { description: 'Amount of entities processed' }, + ); + + const processingDuration = meter.createHistogram( + 'catalog.processing.duration', + { + description: 'Time spent executing the full processing flow', + unit: 'seconds', + }, + ); + + const processorsDuration = meter.createHistogram( + 'catalog.processors.duration', + { + description: 'Time spent executing catalog processors', + unit: 'seconds', + }, + ); + + const processingQueueDelay = meter.createHistogram( + 'catalog.processing.queue.delay', + { + description: + 'The amount of delay between being scheduled for processing, and the start of actually being processed', + unit: 'seconds', + }, + ); + function processStart(item: RefreshStateItem, logger: Logger) { + const startTime = process.hrtime(); + const endOverallTimer = promProcessingDuration.startTimer(); + const endProcessorsTimer = promProcessorsDuration.startTimer(); + logger.debug(`Processing ${item.entityRef}`); if (item.nextUpdateAt) { - processingQueueDelay.observe(-item.nextUpdateAt.diffNow().as('seconds')); + promProcessingQueueDelay.observe( + -item.nextUpdateAt.diffNow().as('seconds'), + ); + processingQueueDelay.record(-item.nextUpdateAt.diffNow().as('seconds')); } - const endOverallTimer = processingDuration.startTimer(); - const endProcessorsTimer = processorsDuration.startTimer(); + function endTime() { + const delta = process.hrtime(startTime); + return delta[0] + delta[1] / 1e9; + } function markProcessorsCompleted(result: EntityProcessingResult) { endProcessorsTimer({ result: result.ok ? 'ok' : 'failed' }); + processorsDuration.record(endTime(), { + result: result.ok ? 'ok' : 'failed', + }); } function markSuccessfulWithNoChanges() { endOverallTimer({ result: 'unchanged' }); - processedEntities.inc({ result: 'unchanged' }, 1); + promProcessedEntities.inc({ result: 'unchanged' }, 1); + + processingDuration.record(endTime(), { result: 'unchanged' }); + processedEntities.add(1, { result: 'unchanged' }); } function markSuccessfulWithErrors() { endOverallTimer({ result: 'errors' }); - processedEntities.inc({ result: 'errors' }, 1); + promProcessedEntities.inc({ result: 'errors' }, 1); + + processingDuration.record(endTime(), { result: 'errors' }); + processedEntities.add(1, { result: 'errors' }); } function markSuccessfulWithChanges(stitchedCount: number) { endOverallTimer({ result: 'changed' }); - stitchedEntities.inc(stitchedCount); - processedEntities.inc({ result: 'changed' }, 1); + promStitchedEntities.inc(stitchedCount); + promProcessedEntities.inc({ result: 'changed' }, 1); + + processingDuration.record(endTime(), { result: 'changed' }); + stitchedEntities.add(stitchedCount); + processedEntities.add(1, { result: 'changed' }); } function markFailed(error: Error) { - processedEntities.inc({ result: 'failed' }, 1); + promProcessedEntities.inc({ result: 'failed' }, 1); + processedEntities.add(1, { result: 'failed' }); logger.warn(`Processing of ${item.entityRef} failed`, error); } diff --git a/yarn.lock b/yarn.lock index 84aaed0d0c..5a4116ed74 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5215,6 +5215,7 @@ __metadata: "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@backstage/types": "workspace:^" + "@opentelemetry/api": ^1.3.0 "@types/core-js": ^2.5.4 "@types/express": ^4.17.6 "@types/git-url-parse": ^9.0.0 @@ -12218,10 +12219,66 @@ __metadata: languageName: node linkType: hard -"@opentelemetry/api@npm:^1.0.1": - version: 1.0.4 - resolution: "@opentelemetry/api@npm:1.0.4" - checksum: 793e9b5c21666b647a60c58c46c3e00ad1dac38505102b026ad0ef617571d637aca54a18533a73c1e288c95b5ac77e2db17f96467f11833ac1165338e1184260 +"@opentelemetry/api@npm:^1.0.1, @opentelemetry/api@npm:^1.3.0": + version: 1.3.0 + resolution: "@opentelemetry/api@npm:1.3.0" + checksum: 33d284b67b6fab20ff72961d289c6487d3cb27caf7489f0231d7030551f82871e081e744b0390751d8aef3bf1614bd79f854788901a354e15274f552581fb374 + languageName: node + linkType: hard + +"@opentelemetry/core@npm:1.8.0": + version: 1.8.0 + resolution: "@opentelemetry/core@npm:1.8.0" + dependencies: + "@opentelemetry/semantic-conventions": 1.8.0 + peerDependencies: + "@opentelemetry/api": ">=1.0.0 <1.4.0" + checksum: 09cd58ec764f97b175af47fbaff335f06a31914fcb59b30c4f96f6ba69391ce4e59e3da46a95fd15a7d0e8fc5c638195aba95ca1a916127207481f1283991c97 + languageName: node + linkType: hard + +"@opentelemetry/exporter-prometheus@npm:^0.34.0": + version: 0.34.0 + resolution: "@opentelemetry/exporter-prometheus@npm:0.34.0" + dependencies: + "@opentelemetry/core": 1.8.0 + "@opentelemetry/resources": 1.8.0 + "@opentelemetry/sdk-metrics": 1.8.0 + peerDependencies: + "@opentelemetry/api": ^1.3.0 + checksum: 6c17d5ec1c638fb5f561230f706d84866d504aa2805ed74ab2174dae9a1c9d39ce909b61ba9796d38fbcdef85f76805e23d5c80d762231e08384e032beb38b65 + languageName: node + linkType: hard + +"@opentelemetry/resources@npm:1.8.0": + version: 1.8.0 + resolution: "@opentelemetry/resources@npm:1.8.0" + dependencies: + "@opentelemetry/core": 1.8.0 + "@opentelemetry/semantic-conventions": 1.8.0 + peerDependencies: + "@opentelemetry/api": ">=1.0.0 <1.4.0" + checksum: eeea7864c486d31679dbae3c31e9badf277ece24ba8bd063e2bcc34b2d7240f543678b8106743aff30c6b0f028f2bed286d64cfd7b5cde6f9a0f6a9271a3fce1 + languageName: node + linkType: hard + +"@opentelemetry/sdk-metrics@npm:1.8.0, @opentelemetry/sdk-metrics@npm:^1.8.0": + version: 1.8.0 + resolution: "@opentelemetry/sdk-metrics@npm:1.8.0" + dependencies: + "@opentelemetry/core": 1.8.0 + "@opentelemetry/resources": 1.8.0 + lodash.merge: 4.6.2 + peerDependencies: + "@opentelemetry/api": ">=1.3.0 <1.4.0" + checksum: 8dfb82e70b14fe2e95ce3f3d0b18e42981bfabe64c63b7f8c429aed197815356836a0350ad5cf38696f561c616ff5572f2ca63f4f968ae1367ed7180f730cad7 + languageName: node + linkType: hard + +"@opentelemetry/semantic-conventions@npm:1.8.0": + version: 1.8.0 + resolution: "@opentelemetry/semantic-conventions@npm:1.8.0" + checksum: df30ad9486b6c611c4110fab80815301a7cc9cb320983d6c5792a1b411dc4e4f04c489b2abfdea0da7f7bbb27b9f3c456764ba07f23432900a9bbcbc5f98ff58 languageName: node linkType: hard @@ -22007,6 +22064,9 @@ __metadata: "@backstage/plugin-todo-backend": "workspace:^" "@gitbeaker/node": ^35.1.0 "@octokit/rest": ^19.0.3 + "@opentelemetry/api": ^1.3.0 + "@opentelemetry/exporter-prometheus": ^0.34.0 + "@opentelemetry/sdk-metrics": ^1.8.0 "@types/dockerode": ^3.3.0 "@types/express": ^4.17.6 "@types/express-serve-static-core": ^4.17.5 @@ -27627,7 +27687,7 @@ __metadata: languageName: node linkType: hard -"lodash.merge@npm:^4.6.2": +"lodash.merge@npm:4.6.2, lodash.merge@npm:^4.6.2": version: 4.6.2 resolution: "lodash.merge@npm:4.6.2" checksum: ad580b4bdbb7ca1f7abf7e1bce63a9a0b98e370cf40194b03380a46b4ed799c9573029599caebc1b14e3f24b111aef72b96674a56cfa105e0f5ac70546cdc005 From 24ff18621cdf08f1a0d806f0f08198c6e0fe3cb1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 9 Dec 2022 17:27:56 +0100 Subject: [PATCH 031/237] catalog-backend: reference cleanup fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: Patrik Oldsberg --- .../database/DefaultProcessingDatabase.test.ts | 17 +++++++++++------ .../src/database/DefaultProcessingDatabase.ts | 3 --- .../src/database/DefaultProviderDatabase.ts | 5 +++++ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 4fbb38b4fc..540c93ef13 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -462,12 +462,17 @@ describe('DefaultProcessingDatabase', () => { knexTx( 'refresh_state_references', ).select(), - ).resolves.toEqual([ - expect.objectContaining({ - source_entity_ref: 'location:default/fakelocation', - target_entity_ref: 'component:default/1', - }), - ]); + ).resolves.toEqual( + step.expectConflict + ? [] + : [ + // eslint-disable-next-line jest/no-conditional-expect + expect.objectContaining({ + source_entity_ref: 'location:default/fakelocation', + target_entity_ref: 'component:default/1', + }), + ], + ); expect(mockLogger.error).not.toHaveBeenCalled(); } diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 1127deed7e..761f5bfd72 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -314,7 +314,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards const stateReferences = new Array(); - const conflictingStateReferences = new Array(); // Upsert all of the unprocessed entities into the refresh_state table, by // their entity ref. @@ -357,13 +356,11 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { this.options.logger.warn( `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`, ); - conflictingStateReferences.push(entityRef); } } // Replace all references for the originating entity or source and then create new ones await tx('refresh_state_references') - .whereNotIn('target_entity_ref', conflictingStateReferences) .andWhere({ source_entity_ref: options.sourceEntityRef }) .delete(); await tx.batchInsert( diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts index f1073fd984..a0d376327b 100644 --- a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts @@ -161,6 +161,11 @@ export class DefaultProviderDatabase implements ProviderDatabase { }); } + await tx('refresh_state_references') + .where('target_entity_ref', entityRef) + .andWhere({ source_key: options.sourceKey }) + .delete(); + if (ok) { await tx( 'refresh_state_references', From 22e51086eb761e9b4354d452ba6084d1c8b49a17 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 9 Dec 2022 17:31:18 +0100 Subject: [PATCH 032/237] catalog-backend: add failing test from #15111 Signed-off-by: Patrik Oldsberg --- .../database/DefaultProviderDatabase.test.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts index 43f2209b60..140925b6fd 100644 --- a/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts @@ -610,6 +610,23 @@ describe('DefaultProviderDatabase', () => { }), ]), ); + let references = await knex( + 'refresh_state_references', + ).select(); + expect(references).toEqual([ + { + id: 1, + source_key: 'lols', + source_entity_ref: null, + target_entity_ref: 'component:default/a', + }, + { + id: 2, + source_key: 'lols', + source_entity_ref: null, + target_entity_ref: 'component:default/b', + }, + ]); await db.transaction(async tx => { await db.replaceUnprocessedEntities(tx, { @@ -653,6 +670,23 @@ describe('DefaultProviderDatabase', () => { }), ]), ); + references = await knex( + 'refresh_state_references', + ).select(); + expect(references).toEqual([ + { + id: 2, + source_key: 'lols', + source_entity_ref: null, + target_entity_ref: 'component:default/b', + }, + { + id: 3, + source_key: 'lols', + source_entity_ref: null, + target_entity_ref: 'component:default/a', + }, + ]); }, 60_000, ); From d136793ff0abe67b4d43d16ccfa1b14cfaf72f69 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 9 Dec 2022 17:32:25 +0100 Subject: [PATCH 033/237] changesets: added changeset for catalog reference fix Signed-off-by: Patrik Oldsberg --- .changeset/tasty-impalas-mix.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/tasty-impalas-mix.md diff --git a/.changeset/tasty-impalas-mix.md b/.changeset/tasty-impalas-mix.md new file mode 100644 index 0000000000..c830c45c56 --- /dev/null +++ b/.changeset/tasty-impalas-mix.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Fixed an issue where internal references in the catalog would stick around for longer than expected, causing entities to not be deleted or orphaned as expected. From 7da60b5b4f5c9df18b524ad2d0957121d2b3cbed Mon Sep 17 00:00:00 2001 From: iris Date: Mon, 12 Dec 2022 12:36:16 +0800 Subject: [PATCH 034/237] revert changes about adding backstage.io/last_updated_at Signed-off-by: iris --- .../src/service/DefaultEntitiesCatalog.ts | 25 +++---------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index a305df52e3..14f22f862b 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -210,14 +210,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { }; } - let entities: Entity[] = rows.map(e => { - const entityJson = JSON.parse(e.final_entity!); - if (e.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated_at'] = - e.last_updated_at; - } - return entityJson; - }); + let entities: Entity[] = rows.map(e => JSON.parse(e.final_entity!)); if (request?.fields) { entities = entities.map(e => request.fields!(e)); @@ -270,12 +263,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { query = parseFilter(request.filter, query, this.database); } for (const row of await query) { - const entityJson = JSON.parse(row.entity); - if (row.entity.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated_at'] = - row.entity.last_updated_at; - } - lookup.set(row.entityRef, row.entity ? entityJson : null); + lookup.set(row.entityRef, row.entity ? JSON.parse(row.entity) : null); } } @@ -385,18 +373,13 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { .where('refresh_state.entity_ref', '=', rootRef) .select({ entityJson: 'final_entities.final_entity', - last_updated_at: 'final_entities.last_updated_at', }); if (!rootRow) { throw new NotFoundError(`No such entity ${rootRef}`); } - const entityJson = JSON.parse(rootRow.entityJson); - if (rootRow.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated_at'] = - rootRow.last_updated_at; - } - const rootEntity = entityJson as Entity; + + const rootEntity = JSON.parse(rootRow.entityJson) as Entity; const seenEntityRefs = new Set(); const todo = new Array(); const items = new Array<{ entity: Entity; parentEntityRefs: string[] }>(); From 1003f52b19a952cb5c298f7c39cfb8ba2bfeb3d4 Mon Sep 17 00:00:00 2001 From: iris Date: Mon, 12 Dec 2022 15:05:43 +0800 Subject: [PATCH 035/237] Change last_updated_at to dateTime Signed-off-by: iris --- .../20221201085245_add_last_updated_at_in_final_entities.js | 2 +- plugins/catalog-backend/src/database/tables.ts | 2 +- plugins/catalog-backend/src/stitching/Stitcher.ts | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js index d12d5bec3e..e34d45ade6 100644 --- a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js +++ b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js @@ -17,7 +17,7 @@ exports.up = async function up(knex) { await knex.schema.table('final_entities', table => { table - .bigint('last_updated_at') + .dateTime('last_updated_at') .nullable() .comment('The time when final_entity changed'); }); diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index 2778cf2fba..9d89b17d16 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -66,7 +66,7 @@ export type DbFinalEntitiesRow = { hash: string; stitch_ticket: string; final_entity?: string; - last_updated_at: string | null; + last_updated_at: string | Date; }; export type DbSearchRow = { diff --git a/plugins/catalog-backend/src/stitching/Stitcher.ts b/plugins/catalog-backend/src/stitching/Stitcher.ts index c3892244dd..4742c81933 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.ts @@ -23,7 +23,6 @@ import { import { SerializedError, stringifyError } from '@backstage/errors'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; -import { DateTime } from 'luxon'; import { Logger } from 'winston'; import { DbFinalEntitiesRow, @@ -208,12 +207,12 @@ export class Stitcher { .update({ final_entity: JSON.stringify(entity), hash, - last_updated_at: `${DateTime.now().toMillis()}`, + last_updated_at: this.database.fn.now(), }) .where('entity_id', entityId) .where('stitch_ticket', ticket) .onConflict('entity_id') - .merge(['final_entity', 'hash']); + .merge(['final_entity', 'hash', 'last_updated_at']); if (amountOfRowsChanged === 0) { this.logger.debug( From 9af905d5d4be8700f4476793cfc2d50b457effbf Mon Sep 17 00:00:00 2001 From: iris Date: Mon, 12 Dec 2022 15:09:48 +0800 Subject: [PATCH 036/237] update test case Signed-off-by: iris --- .changeset/lucky-singers-worry.md | 2 +- .../src/stitching/Stitcher.test.ts | 18 ------------------ 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/.changeset/lucky-singers-worry.md b/.changeset/lucky-singers-worry.md index abe581fbc3..4707a57525 100644 --- a/.changeset/lucky-singers-worry.md +++ b/.changeset/lucky-singers-worry.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend': major --- -Track the last time the final entity changed with new timestamp "last updated at" data in final entities database, which gets updated with the time when final entity is updated. And it's displayed in metadata annotation. +Track the last time the final entity changed with new timestamp "last updated at" data in final entities database, which gets updated with the time when final entity is updated. diff --git a/plugins/catalog-backend/src/stitching/Stitcher.test.ts b/plugins/catalog-backend/src/stitching/Stitcher.test.ts index a13467e9bd..31a8e04392 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.test.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.test.ts @@ -17,7 +17,6 @@ import { getVoidLogger } from '@backstage/backend-common'; import { TestDatabases } from '@backstage/backend-test-utils'; import { Entity } from '@backstage/catalog-model'; -import { DateTime } from 'luxon'; import { applyDatabaseMigrations } from '../database/migrations'; import { DbFinalEntitiesRow, @@ -43,7 +42,6 @@ describe('Stitcher', () => { const stitcher = new Stitcher(db, logger); let entities: DbFinalEntitiesRow[]; let entity: Entity; - const timeBeforeStitch = DateTime.now(); await db('refresh_state').insert([ { @@ -114,13 +112,6 @@ describe('Stitcher', () => { expect(entity.metadata.etag).toEqual(entities[0].hash); const last_updated_at = entities[0].last_updated_at; expect(last_updated_at).not.toBeNull(); - const lastUpdatedAt = DateTime.fromMillis( - last_updated_at ? +last_updated_at : 0, - ); - const msAfterStitch = lastUpdatedAt - .diff(timeBeforeStitch, 'milliseconds') - .toObject(); - expect(msAfterStitch.milliseconds).toBeGreaterThan(0); const firstHash = entities[0].hash; const search = await db('search'); @@ -190,7 +181,6 @@ describe('Stitcher', () => { }, ]); - const timeBeforeRestitch = DateTime.now(); await stitcher.stitch(new Set(['k:ns/n'])); entities = await db('final_entities'); @@ -224,14 +214,6 @@ describe('Stitcher', () => { expect(entities[0].hash).not.toEqual(firstHash); expect(entities[0].hash).toEqual(entity.metadata.etag); expect(entity.metadata.etag).toEqual(entities[0].hash); - const last_updated_at_after_restitch = entities[0].last_updated_at; - expect(last_updated_at_after_restitch).not.toBeNull(); - const msAfterRestitch = DateTime.fromMillis( - last_updated_at_after_restitch ? +last_updated_at_after_restitch : 0, - ) - .diff(timeBeforeRestitch, 'milliseconds') - .toObject(); - expect(msAfterRestitch.milliseconds).toBeGreaterThan(0); expect(await db('search')).toEqual( expect.arrayContaining([ From fc52930f3ec2b3afa665610b0bae706996098fb3 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 12 Dec 2022 09:41:56 +0100 Subject: [PATCH 037/237] fix yarn.lock Signed-off-by: Johan Haals --- yarn.lock | 61 +------------------------------------------------------ 1 file changed, 1 insertion(+), 60 deletions(-) diff --git a/yarn.lock b/yarn.lock index 5a4116ed74..88c1a86ce3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12226,62 +12226,6 @@ __metadata: languageName: node linkType: hard -"@opentelemetry/core@npm:1.8.0": - version: 1.8.0 - resolution: "@opentelemetry/core@npm:1.8.0" - dependencies: - "@opentelemetry/semantic-conventions": 1.8.0 - peerDependencies: - "@opentelemetry/api": ">=1.0.0 <1.4.0" - checksum: 09cd58ec764f97b175af47fbaff335f06a31914fcb59b30c4f96f6ba69391ce4e59e3da46a95fd15a7d0e8fc5c638195aba95ca1a916127207481f1283991c97 - languageName: node - linkType: hard - -"@opentelemetry/exporter-prometheus@npm:^0.34.0": - version: 0.34.0 - resolution: "@opentelemetry/exporter-prometheus@npm:0.34.0" - dependencies: - "@opentelemetry/core": 1.8.0 - "@opentelemetry/resources": 1.8.0 - "@opentelemetry/sdk-metrics": 1.8.0 - peerDependencies: - "@opentelemetry/api": ^1.3.0 - checksum: 6c17d5ec1c638fb5f561230f706d84866d504aa2805ed74ab2174dae9a1c9d39ce909b61ba9796d38fbcdef85f76805e23d5c80d762231e08384e032beb38b65 - languageName: node - linkType: hard - -"@opentelemetry/resources@npm:1.8.0": - version: 1.8.0 - resolution: "@opentelemetry/resources@npm:1.8.0" - dependencies: - "@opentelemetry/core": 1.8.0 - "@opentelemetry/semantic-conventions": 1.8.0 - peerDependencies: - "@opentelemetry/api": ">=1.0.0 <1.4.0" - checksum: eeea7864c486d31679dbae3c31e9badf277ece24ba8bd063e2bcc34b2d7240f543678b8106743aff30c6b0f028f2bed286d64cfd7b5cde6f9a0f6a9271a3fce1 - languageName: node - linkType: hard - -"@opentelemetry/sdk-metrics@npm:1.8.0, @opentelemetry/sdk-metrics@npm:^1.8.0": - version: 1.8.0 - resolution: "@opentelemetry/sdk-metrics@npm:1.8.0" - dependencies: - "@opentelemetry/core": 1.8.0 - "@opentelemetry/resources": 1.8.0 - lodash.merge: 4.6.2 - peerDependencies: - "@opentelemetry/api": ">=1.3.0 <1.4.0" - checksum: 8dfb82e70b14fe2e95ce3f3d0b18e42981bfabe64c63b7f8c429aed197815356836a0350ad5cf38696f561c616ff5572f2ca63f4f968ae1367ed7180f730cad7 - languageName: node - linkType: hard - -"@opentelemetry/semantic-conventions@npm:1.8.0": - version: 1.8.0 - resolution: "@opentelemetry/semantic-conventions@npm:1.8.0" - checksum: df30ad9486b6c611c4110fab80815301a7cc9cb320983d6c5792a1b411dc4e4f04c489b2abfdea0da7f7bbb27b9f3c456764ba07f23432900a9bbcbc5f98ff58 - languageName: node - linkType: hard - "@oriflame/backstage-plugin-score-card@npm:^0.5.1": version: 0.5.7 resolution: "@oriflame/backstage-plugin-score-card@npm:0.5.7" @@ -22064,9 +22008,6 @@ __metadata: "@backstage/plugin-todo-backend": "workspace:^" "@gitbeaker/node": ^35.1.0 "@octokit/rest": ^19.0.3 - "@opentelemetry/api": ^1.3.0 - "@opentelemetry/exporter-prometheus": ^0.34.0 - "@opentelemetry/sdk-metrics": ^1.8.0 "@types/dockerode": ^3.3.0 "@types/express": ^4.17.6 "@types/express-serve-static-core": ^4.17.5 @@ -27687,7 +27628,7 @@ __metadata: languageName: node linkType: hard -"lodash.merge@npm:4.6.2, lodash.merge@npm:^4.6.2": +"lodash.merge@npm:^4.6.2": version: 4.6.2 resolution: "lodash.merge@npm:4.6.2" checksum: ad580b4bdbb7ca1f7abf7e1bce63a9a0b98e370cf40194b03380a46b4ed799c9573029599caebc1b14e3f24b111aef72b96674a56cfa105e0f5ac70546cdc005 From 9cfb9842c89fc95095c6f6ab547c303f401dd30e Mon Sep 17 00:00:00 2001 From: Djam Date: Mon, 12 Dec 2022 13:47:58 +0100 Subject: [PATCH 038/237] Update automate_changeset_feedback.yml Signed-off-by: Djam --- .github/workflows/automate_changeset_feedback.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/automate_changeset_feedback.yml b/.github/workflows/automate_changeset_feedback.yml index 3637ed11e7..dc1b22fc8c 100644 --- a/.github/workflows/automate_changeset_feedback.yml +++ b/.github/workflows/automate_changeset_feedback.yml @@ -26,7 +26,6 @@ jobs: # Fetch the commit that's merged into the base rather than the target ref # This will let us diff only the contents of the PR, without fetching more history diffRef: 'refs/pull/${{ github.event.pull_request.number }}/merge' - github-token: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }} issue-number: ${{ steps.pr-number.outputs.pr-number }} app-id: ${{ secrets.BACKSTAGE_GOALIE_APPLICATION_ID }} private-key: ${{ secrets.BACKSTAGE_GOALIE_PRIVATE_KEY }} From 17d7c2e11a9e51afb7eda1319a2149eb5dd907f1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 12 Dec 2022 14:31:54 +0000 Subject: [PATCH 039/237] Update dependency @swc/jest to v0.2.24 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 1104c7058a..9b8239ad49 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13266,14 +13266,14 @@ __metadata: linkType: hard "@swc/jest@npm:^0.2.22": - version: 0.2.23 - resolution: "@swc/jest@npm:0.2.23" + version: 0.2.24 + resolution: "@swc/jest@npm:0.2.24" dependencies: "@jest/create-cache-key-function": ^27.4.2 jsonc-parser: ^3.2.0 peerDependencies: "@swc/core": "*" - checksum: 1c7db1f6995916ad77369311be078e9d33f2c6a586be9c87927f6a36d124dcd49c29d8c596758cd9dbf4e388ec30f41989e70e574eb59bef3fb41d3131629763 + checksum: 3558213098970cc2882b1f2d1299e78ccea2e18e1e4a4c1820bb669b969ced648eacb14eb78b0bc6fe66e4a60816a7ad7a72c5048ece8382647b8ceac82b708a languageName: node linkType: hard From 496201574ddd1666581d0fe7abe6796281446b5f Mon Sep 17 00:00:00 2001 From: Gabriel Testault Date: Mon, 12 Dec 2022 17:43:23 +0100 Subject: [PATCH 040/237] feature(jenkins-backend): added support for standalone jenkins projects Signed-off-by: Gabriel Testault --- .../src/service/jenkinsApi.test.ts | 31 +++++++++++++++++++ .../jenkins-backend/src/service/jenkinsApi.ts | 14 +++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/plugins/jenkins-backend/src/service/jenkinsApi.test.ts b/plugins/jenkins-backend/src/service/jenkinsApi.test.ts index 4f65e446fd..8413cc6699 100644 --- a/plugins/jenkins-backend/src/service/jenkinsApi.test.ts +++ b/plugins/jenkins-backend/src/service/jenkinsApi.test.ts @@ -75,6 +75,37 @@ describe('JenkinsApi', () => { }, }; + describe('standalone project', () => { + it('should return the only build', async () => { + mockedJenkinsClient.job.get + .mockResolvedValueOnce(project) + .mockResolvedValueOnce(project); + const result = await jenkinsApi.getProjects(jenkinsInfo); + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + actions: [], + displayName: 'Example Build', + fullDisplayName: 'Example jobName » Example Build', + fullName: 'example-jobName/exampleBuild', + inQueue: false, + lastBuild: { + actions: [], + timestamp: 1, + building: false, + duration: 10, + result: 'success', + displayName: '#7', + fullDisplayName: 'Example jobName » Example Build #7', + url: 'https://jenkins.example.com/job/example-jobName/job/exampleBuild', + number: 7, + status: 'success', + source: {}, + }, + status: 'success', + }); + }); + }); + describe('unfiltered', () => { it('standard github layout', async () => { mockedJenkinsClient.job.get.mockResolvedValueOnce({ jobs: [project] }); diff --git a/plugins/jenkins-backend/src/service/jenkinsApi.ts b/plugins/jenkins-backend/src/service/jenkinsApi.ts index 89c9a41f9c..be8fe1b983 100644 --- a/plugins/jenkins-backend/src/service/jenkinsApi.ts +++ b/plugins/jenkins-backend/src/service/jenkinsApi.ts @@ -89,7 +89,7 @@ export class JenkinsApiImpl { } else { // We aren't filtering // Assume jenkinsInfo.jobFullName is a folder which contains one job per branch. - const folder = await client.job.get({ + const project = await client.job.get({ name: jenkinsInfo.jobFullName, // Filter only be the information we need, instead of loading all fields. // Limit to only show the latest build for each job and only load 50 jobs @@ -99,8 +99,16 @@ export class JenkinsApiImpl { tree: JenkinsApiImpl.jobsTreeSpec.replace(/\s/g, ''), }); - // TODO: support this being a project itself. - for (const jobDetails of folder.jobs) { + const isStandaloneProject = !project.jobs; + if (isStandaloneProject) { + const standaloneProject = await client.job.get({ + name: jenkinsInfo.jobFullName, + tree: JenkinsApiImpl.jobTreeSpec.replace(/\s/g, ''), + }); + projects.push(this.augmentProject(standaloneProject)); + return projects; + } + for (const jobDetails of project.jobs) { // for each branch (we assume) if (jobDetails?.jobs) { // skipping folders inside folders for now From 9447b0fb4659e78ad2dc40d2193d269a3be06230 Mon Sep 17 00:00:00 2001 From: Gabriel Testault Date: Mon, 12 Dec 2022 17:46:14 +0100 Subject: [PATCH 041/237] feature(jenkins-backend): add changeset Signed-off-by: Gabriel Testault --- .changeset/seven-dolphins-tickle.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/seven-dolphins-tickle.md diff --git a/.changeset/seven-dolphins-tickle.md b/.changeset/seven-dolphins-tickle.md new file mode 100644 index 0000000000..b3b7697dd6 --- /dev/null +++ b/.changeset/seven-dolphins-tickle.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-jenkins-backend': minor +--- + +added support for standalone jenkins projects From 9a8325aca640dd42980507b6a56cbdb077ae26e3 Mon Sep 17 00:00:00 2001 From: Esther Annorzie Date: Mon, 12 Dec 2022 14:11:11 -0500 Subject: [PATCH 042/237] Make noStarredEntitiesMessage a Content prop Signed-off-by: Esther Annorzie --- .../src/homePageComponents/StarredEntities/Content.tsx | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/plugins/home/src/homePageComponents/StarredEntities/Content.tsx b/plugins/home/src/homePageComponents/StarredEntities/Content.tsx index 1f31ac062f..4654a7f860 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/Content.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/Content.tsx @@ -42,11 +42,9 @@ import useAsync from 'react-use/lib/useAsync'; * @public */ -interface starredEntitiesProp { - noStarredEntitiesMessage?: React.ReactNode; -} - -export const Content = ({ noStarredEntitiesMessage }: starredEntitiesProp) => { +export const Content = (props: { + noStarredEntitiesMessage?: React.ReactNode | undefined; +}) => { const catalogApi = useApi(catalogApiRef); const catalogEntityRoute = useRouteRef(entityRouteRef); const { starredEntities, toggleStarredEntity } = useStarredEntities(); @@ -81,7 +79,7 @@ export const Content = ({ noStarredEntitiesMessage }: starredEntitiesProp) => { if (starredEntities.size === 0) return ( - {noStarredEntitiesMessage || + {props.noStarredEntitiesMessage || 'Click the star beside an entity name to add it to this list!'} ); From 2d4f7f0a158b97144d49286363c4b17724bcb7cf Mon Sep 17 00:00:00 2001 From: Michael Dunton Date: Mon, 12 Dec 2022 14:46:25 -0500 Subject: [PATCH 043/237] Fix Node-Postgres config options URL While following along with the getting started documentation, I wanted to change the user provided to the connection string. When using the link in the comment it took my to a 404 page. I found the correct link. Signed-off-by: Michael Dunton --- docs/getting-started/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index fc5b02e6f4..0441d3f589 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -79,7 +79,7 @@ backend: database: - client: better-sqlite3 - connection: ':memory:' -+ # config options: https://node-postgres.com/api/client ++ # config options: https://node-postgres.com/apis/client + client: pg + connection: + host: ${POSTGRES_HOST} From e2215bc070b5b1256b73492df3ad69c55242f9e3 Mon Sep 17 00:00:00 2001 From: iris Date: Tue, 13 Dec 2022 10:48:00 +0800 Subject: [PATCH 044/237] update changeset to patch Signed-off-by: iris --- .changeset/lucky-singers-worry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/lucky-singers-worry.md b/.changeset/lucky-singers-worry.md index 4707a57525..abefe2bea2 100644 --- a/.changeset/lucky-singers-worry.md +++ b/.changeset/lucky-singers-worry.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-backend': major +'@backstage/plugin-catalog-backend': patch --- Track the last time the final entity changed with new timestamp "last updated at" data in final entities database, which gets updated with the time when final entity is updated. From 24b74c35792742327763e2e1513b4df643ee155d Mon Sep 17 00:00:00 2001 From: iris Date: Tue, 13 Dec 2022 10:51:46 +0800 Subject: [PATCH 045/237] Remove duplicate check Signed-off-by: iris --- plugins/catalog-backend/src/stitching/Stitcher.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/catalog-backend/src/stitching/Stitcher.test.ts b/plugins/catalog-backend/src/stitching/Stitcher.test.ts index 31a8e04392..729793a8d7 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.test.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.test.ts @@ -213,7 +213,6 @@ describe('Stitcher', () => { expect(entities[0].hash).not.toEqual(firstHash); expect(entities[0].hash).toEqual(entity.metadata.etag); - expect(entity.metadata.etag).toEqual(entities[0].hash); expect(await db('search')).toEqual( expect.arrayContaining([ From 434b4ced60f70ca5e126d4b7381f5e10d28eb99b Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 13 Dec 2022 10:40:59 +0100 Subject: [PATCH 046/237] chore: count seconds once Signed-off-by: Johan Haals --- .../src/processing/DefaultCatalogProcessingEngine.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index a27e5a0ce4..4c10460732 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -329,10 +329,9 @@ function progressTracker() { logger.debug(`Processing ${item.entityRef}`); if (item.nextUpdateAt) { - promProcessingQueueDelay.observe( - -item.nextUpdateAt.diffNow().as('seconds'), - ); - processingQueueDelay.record(-item.nextUpdateAt.diffNow().as('seconds')); + const seconds = -item.nextUpdateAt.diffNow().as('seconds'); + promProcessingQueueDelay.observe(seconds); + processingQueueDelay.record(seconds); } function endTime() { From eafc9e7aaecf896d44faf4c67b6cd2ef0e26948e Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 13 Dec 2022 10:49:02 +0100 Subject: [PATCH 047/237] chore: fix metric name typo Signed-off-by: Johan Haals --- .../src/processing/DefaultCatalogProcessingEngine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index 4c10460732..c8f159b195 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -292,7 +292,7 @@ function progressTracker() { ); const processedEntities = meter.createCounter( - 'catalog.stitched.entities.count', + 'catalog.processed.entities.count', { description: 'Amount of entities processed' }, ); From 99656a9e1ab5773f7d22ab34a1c5d7ac92e775d9 Mon Sep 17 00:00:00 2001 From: djamaile Date: Tue, 13 Dec 2022 14:51:00 +0100 Subject: [PATCH 048/237] fix: include asset types for repo tools package Signed-off-by: djamaile --- .../src/commands/api-reports/api-extractor.ts | 3 +- .../api-reports/asset-types/asset-types.d.ts | 131 ++++++++++++++++++ 2 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 packages/repo-tools/src/commands/api-reports/asset-types/asset-types.d.ts diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index 46f4ea984b..2fba8eaf76 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -232,7 +232,8 @@ export async function createTemporaryTsConfig(includedPackageDirs: string[]) { extends: './tsconfig.json', include: [ // These two contain global definitions that are needed for stable API report generation - 'packages/cli/asset-types/asset-types.d.ts', + // eslint-disable-next-line no-restricted-syntax + `${resolvePath(__dirname)}/asset-types/asset-types.d.ts`, ...includedPackageDirs.map(dir => join(dir, 'src')), ], }); diff --git a/packages/repo-tools/src/commands/api-reports/asset-types/asset-types.d.ts b/packages/repo-tools/src/commands/api-reports/asset-types/asset-types.d.ts new file mode 100644 index 0000000000..0bd0922004 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/asset-types/asset-types.d.ts @@ -0,0 +1,131 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable import/no-extraneous-dependencies */ + +/// +/// +/// + +declare module '*.bmp' { + const src: string; + export default src; +} + +declare module '*.gif' { + const src: string; + export default src; +} + +declare module '*.jpg' { + const src: string; + export default src; +} + +declare module '*.jpeg' { + const src: string; + export default src; +} + +declare module '*.png' { + const src: string; + export default src; +} + +declare module '*.webp' { + const src: string; + export default src; +} + +declare module '*.yaml' { + const src: string; + export default src; +} + +declare module '*.icon.svg' { + import { ComponentType } from 'react'; + import { SvgIconProps } from '@material-ui/core'; + + const Icon: ComponentType; + export default Icon; +} + +declare module '*.svg' { + const src: string; + export default src; +} + +declare module '*.eot' { + const src: string; + export default src; +} + +declare module '*.woff' { + const src: string; + export default src; +} + +declare module '*.woff2' { + const src: string; + export default src; +} + +declare module '*.ttf' { + const src: string; + export default src; +} + +declare module '*.css' { + const classes: { readonly [key: string]: string }; + export default classes; +} + +declare module '*.scss' { + const classes: { readonly [key: string]: string }; + export default classes; +} + +declare module '*.sass' { + const classes: { readonly [key: string]: string }; + export default classes; +} + +declare module '*.module.css' { + const classes: { readonly [key: string]: string }; + export default classes; +} + +declare module '*.module.scss' { + const classes: { readonly [key: string]: string }; + export default classes; +} + +declare module '*.module.sass' { + const classes: { readonly [key: string]: string }; + export default classes; +} + +// NOTE(freben): Both the fix, and the placement of the fix, are not great. +// +// The fix is because the PositionError was renamed to +// GeolocationPositionError outside of our control, and react-use is dependent +// on the old name. +// +// The placement is because it's the one location we have at the moment, where +// a central .d.ts file is imported by the frontend and can be amended. +// +// After both TS and react-use are bumped high enough, this should be removed. +type PositionError = GeolocationPositionError; From 25ec5c0c3a5204d59c8f0cc6963d52f01e55e05e Mon Sep 17 00:00:00 2001 From: djamaile Date: Tue, 13 Dec 2022 14:54:30 +0100 Subject: [PATCH 049/237] fix: add changeset Signed-off-by: djamaile --- .changeset/soft-lies-watch.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/soft-lies-watch.md diff --git a/.changeset/soft-lies-watch.md b/.changeset/soft-lies-watch.md new file mode 100644 index 0000000000..6e38e0f1fa --- /dev/null +++ b/.changeset/soft-lies-watch.md @@ -0,0 +1,5 @@ +--- +'@backstage/repo-tools': patch +--- + +Include asset-types.d.ts while running the api report command From b21d440cdb659ecae60bfba24c34e43bd0bb5617 Mon Sep 17 00:00:00 2001 From: djamaile Date: Tue, 13 Dec 2022 14:59:17 +0100 Subject: [PATCH 050/237] fix: use resolvePath instead Signed-off-by: djamaile --- .../repo-tools/src/commands/api-reports/api-extractor.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index 2fba8eaf76..cd69251594 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -66,6 +66,7 @@ import { import { IMarkdownEmitterContext } from '@microsoft/api-documenter/lib/markdown/MarkdownEmitter'; import { AstDeclaration } from '@microsoft/api-extractor/lib/analyzer/AstDeclaration'; import { paths as cliPaths } from '../../lib/paths'; +import { resolvePackagePath as resolvePackagePathBackend } from '@backstage/backend-common'; import minimatch from 'minimatch'; @@ -228,12 +229,13 @@ export async function createTemporaryTsConfig(includedPackageDirs: string[]) { fs.removeSync(path); }); + console.log(resolvePackagePathBackend('@backstage/cli', 'asset-types/asset-types.d.ts')); + await fs.writeJson(path, { extends: './tsconfig.json', include: [ // These two contain global definitions that are needed for stable API report generation - // eslint-disable-next-line no-restricted-syntax - `${resolvePath(__dirname)}/asset-types/asset-types.d.ts`, + `${resolvePackagePathBackend('@backstage/cli', 'asset-types/asset-types.d.ts')}`, ...includedPackageDirs.map(dir => join(dir, 'src')), ], }); From 713100a2c02351f36e82f34788d5ac3a3c438472 Mon Sep 17 00:00:00 2001 From: djamaile Date: Tue, 13 Dec 2022 15:00:47 +0100 Subject: [PATCH 051/237] chore: remove clutter Signed-off-by: djamaile --- .../src/commands/api-reports/api-extractor.ts | 2 - .../api-reports/asset-types/asset-types.d.ts | 131 ------------------ 2 files changed, 133 deletions(-) delete mode 100644 packages/repo-tools/src/commands/api-reports/asset-types/asset-types.d.ts diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index cd69251594..24999a0257 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -229,8 +229,6 @@ export async function createTemporaryTsConfig(includedPackageDirs: string[]) { fs.removeSync(path); }); - console.log(resolvePackagePathBackend('@backstage/cli', 'asset-types/asset-types.d.ts')); - await fs.writeJson(path, { extends: './tsconfig.json', include: [ diff --git a/packages/repo-tools/src/commands/api-reports/asset-types/asset-types.d.ts b/packages/repo-tools/src/commands/api-reports/asset-types/asset-types.d.ts deleted file mode 100644 index 0bd0922004..0000000000 --- a/packages/repo-tools/src/commands/api-reports/asset-types/asset-types.d.ts +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* eslint-disable import/no-extraneous-dependencies */ - -/// -/// -/// - -declare module '*.bmp' { - const src: string; - export default src; -} - -declare module '*.gif' { - const src: string; - export default src; -} - -declare module '*.jpg' { - const src: string; - export default src; -} - -declare module '*.jpeg' { - const src: string; - export default src; -} - -declare module '*.png' { - const src: string; - export default src; -} - -declare module '*.webp' { - const src: string; - export default src; -} - -declare module '*.yaml' { - const src: string; - export default src; -} - -declare module '*.icon.svg' { - import { ComponentType } from 'react'; - import { SvgIconProps } from '@material-ui/core'; - - const Icon: ComponentType; - export default Icon; -} - -declare module '*.svg' { - const src: string; - export default src; -} - -declare module '*.eot' { - const src: string; - export default src; -} - -declare module '*.woff' { - const src: string; - export default src; -} - -declare module '*.woff2' { - const src: string; - export default src; -} - -declare module '*.ttf' { - const src: string; - export default src; -} - -declare module '*.css' { - const classes: { readonly [key: string]: string }; - export default classes; -} - -declare module '*.scss' { - const classes: { readonly [key: string]: string }; - export default classes; -} - -declare module '*.sass' { - const classes: { readonly [key: string]: string }; - export default classes; -} - -declare module '*.module.css' { - const classes: { readonly [key: string]: string }; - export default classes; -} - -declare module '*.module.scss' { - const classes: { readonly [key: string]: string }; - export default classes; -} - -declare module '*.module.sass' { - const classes: { readonly [key: string]: string }; - export default classes; -} - -// NOTE(freben): Both the fix, and the placement of the fix, are not great. -// -// The fix is because the PositionError was renamed to -// GeolocationPositionError outside of our control, and react-use is dependent -// on the old name. -// -// The placement is because it's the one location we have at the moment, where -// a central .d.ts file is imported by the frontend and can be amended. -// -// After both TS and react-use are bumped high enough, this should be removed. -type PositionError = GeolocationPositionError; From dc7123d1e38e8f0617ce390fd4b559ad8a106e4c Mon Sep 17 00:00:00 2001 From: djamaile Date: Tue, 13 Dec 2022 15:02:32 +0100 Subject: [PATCH 052/237] chore: remove string ip Signed-off-by: djamaile --- packages/repo-tools/src/commands/api-reports/api-extractor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index 24999a0257..c14a90152f 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -233,7 +233,7 @@ export async function createTemporaryTsConfig(includedPackageDirs: string[]) { extends: './tsconfig.json', include: [ // These two contain global definitions that are needed for stable API report generation - `${resolvePackagePathBackend('@backstage/cli', 'asset-types/asset-types.d.ts')}`, + resolvePackagePathBackend('@backstage/cli', 'asset-types/asset-types.d.ts'), ...includedPackageDirs.map(dir => join(dir, 'src')), ], }); From e8015f78683eb776c12def21a6c98dd789247193 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 13 Dec 2022 15:03:07 +0000 Subject: [PATCH 053/237] Update dependency @rjsf/validator-ajv8 to v5.0.0-beta.14 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index ca5a89f18e..0fce88c3b1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12741,8 +12741,8 @@ __metadata: linkType: hard "@rjsf/validator-ajv8@npm:^5.0.0-beta.12": - version: 5.0.0-beta.13 - resolution: "@rjsf/validator-ajv8@npm:5.0.0-beta.13" + version: 5.0.0-beta.14 + resolution: "@rjsf/validator-ajv8@npm:5.0.0-beta.14" dependencies: ajv-formats: ^2.1.1 ajv8: "npm:ajv@^8.11.0" @@ -12750,7 +12750,7 @@ __metadata: lodash-es: ^4.17.15 peerDependencies: "@rjsf/utils": ^5.0.0-beta.12 - checksum: e0a51e25845745da628794dadef2c7ffc910d26cb99a90fa763d4a5593b554b084c578a73ed688d02defbeb5ddec38066caae527f5dc8dabba7f012cec31e8a3 + checksum: 14f31eabf6bb668953dcedd1c4b2c1dce4d98022600aa511451798a211a0b00a1f50abd861fce8891f7d6dfeb0a0885bcf1686b6d3a7f4add746abe8b58fc8b0 languageName: node linkType: hard From cd1d4171f825bdbf8d6cd3bddf861f9c09c6dc72 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 13 Dec 2022 15:57:20 +0000 Subject: [PATCH 054/237] Update dependency esbuild to v0.16.4 Signed-off-by: Renovate Bot --- yarn.lock | 182 +++++++++++++++++++++++++++--------------------------- 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/yarn.lock b/yarn.lock index 4d60c18102..950ce40db2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9124,9 +9124,9 @@ __metadata: languageName: node linkType: hard -"@esbuild/android-arm64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/android-arm64@npm:0.16.3" +"@esbuild/android-arm64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/android-arm64@npm:0.16.4" conditions: os=android & cpu=arm64 languageName: node linkType: hard @@ -9138,65 +9138,65 @@ __metadata: languageName: node linkType: hard -"@esbuild/android-arm@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/android-arm@npm:0.16.3" +"@esbuild/android-arm@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/android-arm@npm:0.16.4" conditions: os=android & cpu=arm languageName: node linkType: hard -"@esbuild/android-x64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/android-x64@npm:0.16.3" +"@esbuild/android-x64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/android-x64@npm:0.16.4" conditions: os=android & cpu=x64 languageName: node linkType: hard -"@esbuild/darwin-arm64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/darwin-arm64@npm:0.16.3" +"@esbuild/darwin-arm64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/darwin-arm64@npm:0.16.4" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@esbuild/darwin-x64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/darwin-x64@npm:0.16.3" +"@esbuild/darwin-x64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/darwin-x64@npm:0.16.4" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@esbuild/freebsd-arm64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/freebsd-arm64@npm:0.16.3" +"@esbuild/freebsd-arm64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/freebsd-arm64@npm:0.16.4" conditions: os=freebsd & cpu=arm64 languageName: node linkType: hard -"@esbuild/freebsd-x64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/freebsd-x64@npm:0.16.3" +"@esbuild/freebsd-x64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/freebsd-x64@npm:0.16.4" conditions: os=freebsd & cpu=x64 languageName: node linkType: hard -"@esbuild/linux-arm64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/linux-arm64@npm:0.16.3" +"@esbuild/linux-arm64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/linux-arm64@npm:0.16.4" conditions: os=linux & cpu=arm64 languageName: node linkType: hard -"@esbuild/linux-arm@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/linux-arm@npm:0.16.3" +"@esbuild/linux-arm@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/linux-arm@npm:0.16.4" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@esbuild/linux-ia32@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/linux-ia32@npm:0.16.3" +"@esbuild/linux-ia32@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/linux-ia32@npm:0.16.4" conditions: os=linux & cpu=ia32 languageName: node linkType: hard @@ -9208,86 +9208,86 @@ __metadata: languageName: node linkType: hard -"@esbuild/linux-loong64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/linux-loong64@npm:0.16.3" +"@esbuild/linux-loong64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/linux-loong64@npm:0.16.4" conditions: os=linux & cpu=loong64 languageName: node linkType: hard -"@esbuild/linux-mips64el@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/linux-mips64el@npm:0.16.3" +"@esbuild/linux-mips64el@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/linux-mips64el@npm:0.16.4" conditions: os=linux & cpu=mips64el languageName: node linkType: hard -"@esbuild/linux-ppc64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/linux-ppc64@npm:0.16.3" +"@esbuild/linux-ppc64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/linux-ppc64@npm:0.16.4" conditions: os=linux & cpu=ppc64 languageName: node linkType: hard -"@esbuild/linux-riscv64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/linux-riscv64@npm:0.16.3" +"@esbuild/linux-riscv64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/linux-riscv64@npm:0.16.4" conditions: os=linux & cpu=riscv64 languageName: node linkType: hard -"@esbuild/linux-s390x@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/linux-s390x@npm:0.16.3" +"@esbuild/linux-s390x@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/linux-s390x@npm:0.16.4" conditions: os=linux & cpu=s390x languageName: node linkType: hard -"@esbuild/linux-x64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/linux-x64@npm:0.16.3" +"@esbuild/linux-x64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/linux-x64@npm:0.16.4" conditions: os=linux & cpu=x64 languageName: node linkType: hard -"@esbuild/netbsd-x64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/netbsd-x64@npm:0.16.3" +"@esbuild/netbsd-x64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/netbsd-x64@npm:0.16.4" conditions: os=netbsd & cpu=x64 languageName: node linkType: hard -"@esbuild/openbsd-x64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/openbsd-x64@npm:0.16.3" +"@esbuild/openbsd-x64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/openbsd-x64@npm:0.16.4" conditions: os=openbsd & cpu=x64 languageName: node linkType: hard -"@esbuild/sunos-x64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/sunos-x64@npm:0.16.3" +"@esbuild/sunos-x64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/sunos-x64@npm:0.16.4" conditions: os=sunos & cpu=x64 languageName: node linkType: hard -"@esbuild/win32-arm64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/win32-arm64@npm:0.16.3" +"@esbuild/win32-arm64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/win32-arm64@npm:0.16.4" conditions: os=win32 & cpu=arm64 languageName: node linkType: hard -"@esbuild/win32-ia32@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/win32-ia32@npm:0.16.3" +"@esbuild/win32-ia32@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/win32-ia32@npm:0.16.4" conditions: os=win32 & cpu=ia32 languageName: node linkType: hard -"@esbuild/win32-x64@npm:0.16.3": - version: 0.16.3 - resolution: "@esbuild/win32-x64@npm:0.16.3" +"@esbuild/win32-x64@npm:0.16.4": + version: 0.16.4 + resolution: "@esbuild/win32-x64@npm:0.16.4" conditions: os=win32 & cpu=x64 languageName: node linkType: hard @@ -21526,31 +21526,31 @@ __metadata: linkType: hard "esbuild@npm:^0.16.0": - version: 0.16.3 - resolution: "esbuild@npm:0.16.3" + version: 0.16.4 + resolution: "esbuild@npm:0.16.4" dependencies: - "@esbuild/android-arm": 0.16.3 - "@esbuild/android-arm64": 0.16.3 - "@esbuild/android-x64": 0.16.3 - "@esbuild/darwin-arm64": 0.16.3 - "@esbuild/darwin-x64": 0.16.3 - "@esbuild/freebsd-arm64": 0.16.3 - "@esbuild/freebsd-x64": 0.16.3 - "@esbuild/linux-arm": 0.16.3 - "@esbuild/linux-arm64": 0.16.3 - "@esbuild/linux-ia32": 0.16.3 - "@esbuild/linux-loong64": 0.16.3 - "@esbuild/linux-mips64el": 0.16.3 - "@esbuild/linux-ppc64": 0.16.3 - "@esbuild/linux-riscv64": 0.16.3 - "@esbuild/linux-s390x": 0.16.3 - "@esbuild/linux-x64": 0.16.3 - "@esbuild/netbsd-x64": 0.16.3 - "@esbuild/openbsd-x64": 0.16.3 - "@esbuild/sunos-x64": 0.16.3 - "@esbuild/win32-arm64": 0.16.3 - "@esbuild/win32-ia32": 0.16.3 - "@esbuild/win32-x64": 0.16.3 + "@esbuild/android-arm": 0.16.4 + "@esbuild/android-arm64": 0.16.4 + "@esbuild/android-x64": 0.16.4 + "@esbuild/darwin-arm64": 0.16.4 + "@esbuild/darwin-x64": 0.16.4 + "@esbuild/freebsd-arm64": 0.16.4 + "@esbuild/freebsd-x64": 0.16.4 + "@esbuild/linux-arm": 0.16.4 + "@esbuild/linux-arm64": 0.16.4 + "@esbuild/linux-ia32": 0.16.4 + "@esbuild/linux-loong64": 0.16.4 + "@esbuild/linux-mips64el": 0.16.4 + "@esbuild/linux-ppc64": 0.16.4 + "@esbuild/linux-riscv64": 0.16.4 + "@esbuild/linux-s390x": 0.16.4 + "@esbuild/linux-x64": 0.16.4 + "@esbuild/netbsd-x64": 0.16.4 + "@esbuild/openbsd-x64": 0.16.4 + "@esbuild/sunos-x64": 0.16.4 + "@esbuild/win32-arm64": 0.16.4 + "@esbuild/win32-ia32": 0.16.4 + "@esbuild/win32-x64": 0.16.4 dependenciesMeta: "@esbuild/android-arm": optional: true @@ -21598,7 +21598,7 @@ __metadata: optional: true bin: esbuild: bin/esbuild - checksum: c2986b0433c6048b917c185067ea42427413ef4136c45012e180e48fc24e6f01af9c94ca7e9bc6dd29ac529af45d26c9d4eb5b8639c9a79f68f337d24aeda2af + checksum: c06e9b2e84f5c7cdb608fa15e5a241d155321097fe1362beab176bc8283f54ae2a9a7fcca741da2663ffb5fea98c6c47226edd22189d3effb14b457e46592d1b languageName: node linkType: hard From b3fac9c107f59e2354fd6b8c57307113540d2579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 13 Dec 2022 16:59:29 +0100 Subject: [PATCH 055/237] ignore processors emitting entities that are their own children MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/smart-zoos-wash.md | 5 +++++ .../DefaultCatalogProcessingOrchestrator.test.ts | 13 +++++++++++-- .../src/processing/ProcessorOutputCollector.ts | 12 ++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 .changeset/smart-zoos-wash.md diff --git a/.changeset/smart-zoos-wash.md b/.changeset/smart-zoos-wash.md new file mode 100644 index 0000000000..ffb5ca0f01 --- /dev/null +++ b/.changeset/smart-zoos-wash.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Ignore attempts at emitting the current entity as a child of itself. diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts index 9731022582..ceb6992d7a 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts @@ -258,13 +258,22 @@ describe('DefaultCatalogProcessingOrchestrator', () => { }, }; + const child: Entity = { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'Test2', + namespace: 'test1', + }, + }; + it('enforces catalog rules', async () => { const integrations = ScmIntegrations.fromConfig(new ConfigReader({})); const processor: jest.Mocked = { getProcessorName: jest.fn(), validateEntityKind: jest.fn(async () => true), readLocation: jest.fn(async (_l, _o, emit) => { - emit(processingResult.entity({ type: 't', target: 't' }, entity)); + emit(processingResult.entity({ type: 't', target: 't' }, child)); return true; }), }; @@ -300,7 +309,7 @@ describe('DefaultCatalogProcessingOrchestrator', () => { getProcessorName: jest.fn(), validateEntityKind: jest.fn(async () => true), readLocation: jest.fn(async (_l, _o, emit) => { - emit(processingResult.entity({ type: 't', target: 't' }, entity)); + emit(processingResult.entity({ type: 't', target: 't' }, child)); return true; }), }; diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index 19f3b33df4..55282baa38 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -19,6 +19,7 @@ import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, stringifyLocationRef, + stringifyEntityRef, } from '@backstage/catalog-model'; import { assertError } from '@backstage/errors'; import { Logger } from 'winston'; @@ -89,6 +90,17 @@ export class ProcessorOutputCollector { return; } + // The processor contract says you should return the "trunk" (current) + // entity, not emit it. But it happens that this is misunderstood or + // accidentally forgotten. This can lead to circular references which at + // best is wasteful, so we try to be helpful by ignoring such emitted + // entities. + if ( + stringifyEntityRef(entity) === stringifyEntityRef(this.parentEntity) + ) { + return; + } + // Note that at this point, we have only validated the envelope part of // the entity data. Annotations are not part of that, so we have to be // defensive. If the annotations were malformed (e.g. were not a valid From fa85521b1ee6e4b9edf0f6a2e73e7c77df5f75e2 Mon Sep 17 00:00:00 2001 From: Carlos Esteban Lopez Date: Mon, 5 Dec 2022 16:55:17 -0500 Subject: [PATCH 056/237] feat: Enable eslint-plugin-react with recommended settings & forbid button Signed-off-by: Carlos Esteban Lopez --- packages/cli/config/eslint-factory.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/cli/config/eslint-factory.js b/packages/cli/config/eslint-factory.js index a40aa79fb3..ef1305f6d4 100644 --- a/packages/cli/config/eslint-factory.js +++ b/packages/cli/config/eslint-factory.js @@ -215,8 +215,20 @@ function createConfigForRole(dir, role, extraConfig = {}) { ...extraConfig, extends: [ '@spotify/eslint-config-react', + 'plugin:react/recommended', ...(extraConfig.extends ?? []), ], + rules: { + ...extraConfig.rules, + 'react/forbid-elements': [ + 1, + { + forbid: [ + { element: 'button', message: 'use MUI + ); } @@ -688,7 +689,8 @@ export const SidebarExpandButton = () => { }; return ( - + ); }; diff --git a/packages/core-components/src/layout/Sidebar/SidebarSubmenuItem.tsx b/packages/core-components/src/layout/Sidebar/SidebarSubmenuItem.tsx index fbe4496c72..5fdd29f203 100644 --- a/packages/core-components/src/layout/Sidebar/SidebarSubmenuItem.tsx +++ b/packages/core-components/src/layout/Sidebar/SidebarSubmenuItem.tsx @@ -27,6 +27,7 @@ import ArrowDropUpIcon from '@material-ui/icons/ArrowDropUp'; import { SidebarItemWithSubmenuContext } from './config'; import { isLocationMatch } from './utils'; import Box from '@material-ui/core/Box'; +import Button from '@material-ui/core/Button'; const useStyles = makeStyles( theme => ({ @@ -158,7 +159,8 @@ export const SidebarSubmenuItem = (props: SidebarSubmenuItemProps) => { return ( - + {dropdownItems && showDropDown && ( diff --git a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/PreparePullRequestForm.test.tsx b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/PreparePullRequestForm.test.tsx index a8188ce358..54d5c27993 100644 --- a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/PreparePullRequestForm.test.tsx +++ b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/PreparePullRequestForm.test.tsx @@ -15,6 +15,7 @@ */ import { FormHelperText, TextField } from '@material-ui/core'; +import Button from '@material-ui/core/Button'; import { act, render, screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import React from 'react'; @@ -31,7 +32,7 @@ describe('', () => { render={({ register }) => ( <> - {' '} + {' '} )} onSubmit={onSubmitFn} @@ -59,7 +60,7 @@ describe('', () => { id="main" label="Main Field" /> - + )} onSubmit={onSubmitFn} @@ -93,7 +94,7 @@ describe('', () => { Error in required main field )} - {' '} + {' '} )} onSubmit={onSubmitFn} diff --git a/plugins/techdocs/src/reader/transformers/copyToClipboard.tsx b/plugins/techdocs/src/reader/transformers/copyToClipboard.tsx index 6adb4252d3..a90c4ca116 100644 --- a/plugins/techdocs/src/reader/transformers/copyToClipboard.tsx +++ b/plugins/techdocs/src/reader/transformers/copyToClipboard.tsx @@ -23,6 +23,8 @@ import { SvgIcon, Tooltip, } from '@material-ui/core'; +import Button from '@material-ui/core/Button'; +import type { Transformer } from './transformer'; const CopyToClipboardTooltip = withStyles(theme => ({ tooltip: { @@ -65,15 +67,13 @@ const CopyToClipboardButton = ({ text }: CopyToClipboardButtonProps) => { onClose={handleClose} leaveDelay={1000} > - + ); }; -import type { Transformer } from './transformer'; - /** * Recreates copy-to-clipboard functionality attached to snippets that * is native to mkdocs-material theme. diff --git a/plugins/techdocs/src/search/components/TechDocsSearch.test.tsx b/plugins/techdocs/src/search/components/TechDocsSearch.test.tsx index 28d2346c61..05d26b87d8 100644 --- a/plugins/techdocs/src/search/components/TechDocsSearch.test.tsx +++ b/plugins/techdocs/src/search/components/TechDocsSearch.test.tsx @@ -16,6 +16,7 @@ import { ApiProvider } from '@backstage/core-app-api'; import { searchApiRef } from '@backstage/plugin-search-react'; import { TestApiRegistry, wrapInTestApp } from '@backstage/test-utils'; +import Button from '@material-ui/core/Button'; import { act, fireEvent, @@ -136,9 +137,9 @@ describe('', () => { const [entityName, setEntityName] = useState(entityId); return wrapInTestApp( - + , ); From 4cb7d2fbf43ee753067c9687eb49e9c402c6d73a Mon Sep 17 00:00:00 2001 From: djamaile Date: Tue, 13 Dec 2022 17:44:05 +0100 Subject: [PATCH 060/237] chore: add empty exclude Signed-off-by: djamaile --- packages/repo-tools/src/commands/api-reports/api-extractor.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index c14a90152f..63387afe54 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -236,6 +236,8 @@ export async function createTemporaryTsConfig(includedPackageDirs: string[]) { resolvePackagePathBackend('@backstage/cli', 'asset-types/asset-types.d.ts'), ...includedPackageDirs.map(dir => join(dir, 'src')), ], + // we don't exclude node_modules so that we can use the asset-types.d.ts file + exclude: [], }); return path; From 5d3058355ddb901ec9f499eaab1c25e0384b8d8f Mon Sep 17 00:00:00 2001 From: Carlos Esteban Lopez Date: Tue, 13 Dec 2022 11:49:12 -0500 Subject: [PATCH 061/237] chore: Add changeset Signed-off-by: Carlos Esteban Lopez --- .changeset/quick-donkeys-yawn.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/quick-donkeys-yawn.md diff --git a/.changeset/quick-donkeys-yawn.md b/.changeset/quick-donkeys-yawn.md new file mode 100644 index 0000000000..f6b6c23fb9 --- /dev/null +++ b/.changeset/quick-donkeys-yawn.md @@ -0,0 +1,6 @@ +--- +'@backstage/core-components': patch +'@backstage/plugin-techdocs': patch +--- + +Add `react/forbid-elements` linter rule for button, suggest MUI `Button` From 6b261d9b29da56c64b059a5c2321ea1f7a37ade2 Mon Sep 17 00:00:00 2001 From: Alex Rybchenko Date: Tue, 13 Dec 2022 18:05:44 +0100 Subject: [PATCH 062/237] updated changeset Signed-off-by: Alex Rybchenko --- .changeset/nasty-dragons-melt.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.changeset/nasty-dragons-melt.md b/.changeset/nasty-dragons-melt.md index b083ebce62..2f6610ff31 100644 --- a/.changeset/nasty-dragons-melt.md +++ b/.changeset/nasty-dragons-melt.md @@ -1,5 +1,15 @@ --- -'@backstage/plugin-scaffolder': minor +'@backstage/plugin-scaffolder': patch --- -All form data is now passed to validator functions in 'next' scaffolder, so it's now possible to perform validation for fields that depend on other field values +Form data is now passed to validator functions in 'next' scaffolder, so it's now possible to perform validation for fields that depend on other field values. This is something that we discourage due to the coupling that it creates, but is sometimes still the most sensible solution. + +```typescript jsx +export const myCustomValidation = ( + value: string, + validation: FieldValidation, + { apiHolder, formData }: { apiHolder: ApiHolder; formData: JsonObject }, +) => { + // validate +}; +``` From ab3dcebd3a9d8918aedcd400c50aa1a41ef4167b Mon Sep 17 00:00:00 2001 From: djamaile Date: Tue, 13 Dec 2022 18:14:09 +0100 Subject: [PATCH 063/237] chore: run prettier Signed-off-by: djamaile --- .../repo-tools/src/commands/api-reports/api-extractor.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index 63387afe54..43ef750322 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -233,7 +233,10 @@ export async function createTemporaryTsConfig(includedPackageDirs: string[]) { extends: './tsconfig.json', include: [ // These two contain global definitions that are needed for stable API report generation - resolvePackagePathBackend('@backstage/cli', 'asset-types/asset-types.d.ts'), + resolvePackagePathBackend( + '@backstage/cli', + 'asset-types/asset-types.d.ts', + ), ...includedPackageDirs.map(dir => join(dir, 'src')), ], // we don't exclude node_modules so that we can use the asset-types.d.ts file From 26a28bfd24fee41a1f870680c66826ea9f0d0218 Mon Sep 17 00:00:00 2001 From: Andre Wanlin <67169551+awanlin@users.noreply.github.com> Date: Fri, 9 Dec 2022 14:58:59 -0600 Subject: [PATCH 064/237] Make ownership box all the same size Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com> --- .../org/src/components/Cards/OwnershipCard/ComponentsGrid.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/org/src/components/Cards/OwnershipCard/ComponentsGrid.tsx b/plugins/org/src/components/Cards/OwnershipCard/ComponentsGrid.tsx index f996342454..940103382c 100644 --- a/plugins/org/src/components/Cards/OwnershipCard/ComponentsGrid.tsx +++ b/plugins/org/src/components/Cards/OwnershipCard/ComponentsGrid.tsx @@ -75,6 +75,7 @@ const EntityCountTile = ({ display="flex" flexDirection="column" alignItems="center" + style={{ height: '100%' }} > {counter} From 4395eac4d8b33b66360ef72b1159079a696ae527 Mon Sep 17 00:00:00 2001 From: Andre Wanlin <67169551+awanlin@users.noreply.github.com> Date: Fri, 9 Dec 2022 15:02:27 -0600 Subject: [PATCH 065/237] Added changeset Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com> --- .changeset/eighty-melons-walk.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/eighty-melons-walk.md diff --git a/.changeset/eighty-melons-walk.md b/.changeset/eighty-melons-walk.md new file mode 100644 index 0000000000..bf9ee2d78f --- /dev/null +++ b/.changeset/eighty-melons-walk.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-org': patch +--- + +Made all the ownership boxes the same size From 1d3fe791c19aef78b4d14709f2824e1794a80a9c Mon Sep 17 00:00:00 2001 From: Andre Wanlin <67169551+awanlin@users.noreply.github.com> Date: Tue, 13 Dec 2022 13:33:40 -0600 Subject: [PATCH 066/237] Moved style to class Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com> --- .../org/src/components/Cards/OwnershipCard/ComponentsGrid.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/org/src/components/Cards/OwnershipCard/ComponentsGrid.tsx b/plugins/org/src/components/Cards/OwnershipCard/ComponentsGrid.tsx index 940103382c..567c7ad3bc 100644 --- a/plugins/org/src/components/Cards/OwnershipCard/ComponentsGrid.tsx +++ b/plugins/org/src/components/Cards/OwnershipCard/ComponentsGrid.tsx @@ -42,6 +42,7 @@ const useStyles = makeStyles((theme: BackstageTheme) => '&:hover': { boxShadow: theme.shadows[4], }, + height: '100%', }, bold: { fontWeight: theme.typography.fontWeightBold, @@ -75,7 +76,6 @@ const EntityCountTile = ({ display="flex" flexDirection="column" alignItems="center" - style={{ height: '100%' }} > {counter} From 648a20fa0ffd87f5da34abb30b90bc2390d3d74c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 13 Dec 2022 22:16:36 +0100 Subject: [PATCH 067/237] log error too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../DefaultCatalogProcessingOrchestrator.ts | 8 +++--- .../processing/ProcessorOutputCollector.ts | 28 +++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts index 339469ff1d..d177a7d4a8 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts @@ -187,7 +187,7 @@ export class DefaultCatalogProcessingOrchestrator res = await processor.preProcessEntity( res, context.location, - context.collector.onEmit, + context.collector.forProcessor(processor), context.originLocation, context.cache.forProcessor(processor), ); @@ -300,7 +300,7 @@ export class DefaultCatalogProcessingOrchestrator for (const maybeRelativeTarget of targets) { if (type === 'file' && maybeRelativeTarget.endsWith(path.sep)) { - context.collector.onEmit( + context.collector.generic()( processingResult.inputError( context.location, `LocationEntityProcessor cannot handle ${type} type location with target ${context.location.target} that ends with a path separator`, @@ -326,7 +326,7 @@ export class DefaultCatalogProcessingOrchestrator presence, }, presence === 'optional', - context.collector.onEmit, + context.collector.forProcessor(processor), this.options.parser, context.cache.forProcessor(processor, target), ); @@ -365,7 +365,7 @@ export class DefaultCatalogProcessingOrchestrator res = await processor.postProcessEntity( res, context.location, - context.collector.onEmit, + context.collector.forProcessor(processor), context.cache.forProcessor(processor), ); } catch (e) { diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index 55282baa38..0ee6191284 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -24,6 +24,7 @@ import { import { assertError } from '@backstage/errors'; import { Logger } from 'winston'; import { + CatalogProcessor, CatalogProcessorResult, DeferredEntity, EntityRelationSpec, @@ -51,8 +52,17 @@ export class ProcessorOutputCollector { private readonly parentEntity: Entity, ) {} - get onEmit(): (i: CatalogProcessorResult) => void { - return i => this.receive(i); + generic(): (i: CatalogProcessorResult) => void { + return i => this.receive(this.logger, i); + } + + forProcessor( + processor: CatalogProcessor, + ): (i: CatalogProcessorResult) => void { + const logger = this.logger.child({ + processor: processor.getProcessorName(), + }); + return i => this.receive(logger, i); } results() { @@ -65,9 +75,9 @@ export class ProcessorOutputCollector { }; } - private receive(i: CatalogProcessorResult) { + private receive(logger: Logger, i: CatalogProcessorResult) { if (this.done) { - this.logger.warn( + logger.warn( `Item of type "${ i.type }" was emitted after processing had completed. Stack trace: ${ @@ -85,7 +95,7 @@ export class ProcessorOutputCollector { entity = validateEntityEnvelope(i.entity); } catch (e) { assertError(e); - this.logger.debug(`Envelope validation failed at ${location}, ${e}`); + logger.debug(`Envelope validation failed at ${location}, ${e}`); this.errors.push(e); return; } @@ -95,9 +105,11 @@ export class ProcessorOutputCollector { // accidentally forgotten. This can lead to circular references which at // best is wasteful, so we try to be helpful by ignoring such emitted // entities. - if ( - stringifyEntityRef(entity) === stringifyEntityRef(this.parentEntity) - ) { + const entityRef = stringifyEntityRef(entity); + if (entityRef === stringifyEntityRef(this.parentEntity)) { + logger.warn( + `Ignored emitted entity ${entityRef} whose ref was identical to the one being processed. This commonly indicates mistakenly emitting the input entity instead of returning it.`, + ); return; } From 841ce4ffba2dca10d88acddbca0469eb8eec85ce Mon Sep 17 00:00:00 2001 From: Blake Stoddard Date: Wed, 30 Nov 2022 14:01:32 -0500 Subject: [PATCH 068/237] align sidebar icons properly right now they're off because the div is consuming the default mui line height of 1.43 and adding pixels to the bottom of the div Signed-off-by: Blake Stoddard --- packages/core-components/src/layout/Sidebar/Items.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/core-components/src/layout/Sidebar/Items.tsx b/packages/core-components/src/layout/Sidebar/Items.tsx index 031b1e5168..85335bf05a 100644 --- a/packages/core-components/src/layout/Sidebar/Items.tsx +++ b/packages/core-components/src/layout/Sidebar/Items.tsx @@ -140,6 +140,7 @@ const makeSidebarStyles = (sidebarConfig: SidebarConfig) => display: 'flex', alignItems: 'center', justifyContent: 'center', + lineHeight: '0', }, searchRoot: { marginBottom: 12, @@ -371,12 +372,14 @@ const SidebarItemBase = forwardRef((props, ref) => { const { isOpen } = useSidebarOpenState(); const divStyle = - !isOpen && hasSubmenu ? { display: 'flex', marginLeft: '24px' } : {}; + !isOpen && hasSubmenu + ? { display: 'flex', marginLeft: '20px' } + : { lineHeight: '0' }; const displayItemIcon = ( - {!isOpen && hasSubmenu ? : <>} + {!isOpen && hasSubmenu ? : <>} ); From a236a8830dea5e83d992817b6b8fb8037b2e05a5 Mon Sep 17 00:00:00 2001 From: Blake Stoddard Date: Tue, 13 Dec 2022 12:52:57 -0500 Subject: [PATCH 069/237] Add changeset Signed-off-by: Blake Stoddard --- .changeset/dirty-bees-explode.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/dirty-bees-explode.md diff --git a/.changeset/dirty-bees-explode.md b/.changeset/dirty-bees-explode.md new file mode 100644 index 0000000000..0df3bc3b3e --- /dev/null +++ b/.changeset/dirty-bees-explode.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-components': patch +--- + +Update sidebar icon alignment From edf2404e9fb3c1a990dc33bf6d4ee86587f0da03 Mon Sep 17 00:00:00 2001 From: Esther Annorzie Date: Tue, 13 Dec 2022 18:16:08 -0500 Subject: [PATCH 070/237] Add new changeset Signed-off-by: Esther Annorzie --- .changeset/polite-adults-sit.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/polite-adults-sit.md diff --git a/.changeset/polite-adults-sit.md b/.changeset/polite-adults-sit.md new file mode 100644 index 0000000000..3d14a130a5 --- /dev/null +++ b/.changeset/polite-adults-sit.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-home': patch +--- + +Adjusted the description's empty state on the starred entities table, From 554055fc549c373db1123105b85f6a858a07d3b5 Mon Sep 17 00:00:00 2001 From: Esther Annorzie Date: Tue, 13 Dec 2022 18:16:24 -0500 Subject: [PATCH 071/237] Remove old changeset Signed-off-by: Esther Annorzie --- .changeset/many-mangos-behave.md | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 .changeset/many-mangos-behave.md diff --git a/.changeset/many-mangos-behave.md b/.changeset/many-mangos-behave.md deleted file mode 100644 index ff777d3839..0000000000 --- a/.changeset/many-mangos-behave.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@backstage/plugin-home': minor ---- - -'backstage/home': minor - -If no entities are starred, a call to action message displays. From 7ef9bf8c6e90beaa8458e7acb41ccdd8055de742 Mon Sep 17 00:00:00 2001 From: Kyle Leonhard Date: Tue, 13 Dec 2022 15:54:23 -0800 Subject: [PATCH 072/237] Add option to require Github conversation resolution Signed-off-by: Kyle Leonhard --- plugins/scaffolder-backend/api-report.md | 3 +++ .../actions/builtin/github/githubRepoCreate.ts | 3 +++ .../actions/builtin/github/githubRepoPush.test.ts | 15 +++++++++++++++ .../actions/builtin/github/githubRepoPush.ts | 5 +++++ .../scaffolder/actions/builtin/github/helpers.ts | 2 ++ .../actions/builtin/github/inputProperties.ts | 7 +++++++ .../src/scaffolder/actions/builtin/helpers.ts | 3 +++ .../actions/builtin/publish/github.test.ts | 14 ++++++++++++++ .../scaffolder/actions/builtin/publish/github.ts | 5 +++++ 9 files changed, 57 insertions(+) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index fd5af19fe2..40a0ec11a0 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -207,6 +207,7 @@ export function createGithubRepoCreateAction(options: { | undefined; requiredStatusCheckContexts?: string[] | undefined; requireBranchesToBeUpToDate?: boolean | undefined; + requiredConversationResolution?: boolean | undefined; repoVisibility?: 'internal' | 'private' | 'public' | undefined; collaborators?: | ( @@ -253,6 +254,7 @@ export function createGithubRepoPushAction(options: { | undefined; requiredStatusCheckContexts?: string[] | undefined; requireBranchesToBeUpToDate?: boolean | undefined; + requiredConversationResolution?: boolean | undefined; sourcePath?: string | undefined; token?: string | undefined; }>; @@ -392,6 +394,7 @@ export function createPublishGithubAction(options: { dismissStaleReviews?: boolean | undefined; requiredStatusCheckContexts?: string[] | undefined; requireBranchesToBeUpToDate?: boolean | undefined; + requiredConversationResolution?: boolean | undefined; repoVisibility?: 'internal' | 'private' | 'public' | undefined; collaborators?: | ( diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoCreate.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoCreate.ts index 760f42eade..4f2cf4a71e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoCreate.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoCreate.ts @@ -59,6 +59,7 @@ export function createGithubRepoCreateAction(options: { }; requiredStatusCheckContexts?: string[]; requireBranchesToBeUpToDate?: boolean; + requiredConversationResolution?: boolean; repoVisibility?: 'private' | 'internal' | 'public'; collaborators?: Array< | { @@ -93,6 +94,8 @@ export function createGithubRepoCreateAction(options: { bypassPullRequestAllowances: inputProps.bypassPullRequestAllowances, requiredStatusCheckContexts: inputProps.requiredStatusCheckContexts, requireBranchesToBeUpToDate: inputProps.requireBranchesToBeUpToDate, + requiredConversationResolution: + inputProps.requiredConversationResolution, repoVisibility: inputProps.repoVisibility, deleteBranchOnMerge: inputProps.deleteBranchOnMerge, allowMergeCommit: inputProps.allowMergeCommit, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts index 46f33a823d..e7afc26fb8 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts @@ -284,6 +284,7 @@ describe('github:repo:push', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -305,6 +306,7 @@ describe('github:repo:push', () => { requireCodeOwnerReviews: true, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -326,6 +328,7 @@ describe('github:repo:push', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -350,6 +353,7 @@ describe('github:repo:push', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -371,6 +375,7 @@ describe('github:repo:push', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -392,6 +397,7 @@ describe('github:repo:push', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: false, dismissStaleReviews: false, }); @@ -416,6 +422,7 @@ describe('github:repo:push', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -426,6 +433,7 @@ describe('github:repo:push', () => { ...mockContext.input, requiredStatusCheckContexts: ['statusCheck'], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, }, }); @@ -438,6 +446,7 @@ describe('github:repo:push', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: ['statusCheck'], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -460,6 +469,7 @@ describe('github:repo:push', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: ['statusCheck'], requireBranchesToBeUpToDate: false, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -470,6 +480,7 @@ describe('github:repo:push', () => { ...mockContext.input, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, }, }); @@ -482,6 +493,7 @@ describe('github:repo:push', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -525,6 +537,7 @@ describe('github:repo:push', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -546,6 +559,7 @@ describe('github:repo:push', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: true, }); @@ -567,6 +581,7 @@ describe('github:repo:push', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.ts index 4c931e029c..9d470790fa 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.ts @@ -59,6 +59,7 @@ export function createGithubRepoPushAction(options: { | undefined; requiredStatusCheckContexts?: string[]; requireBranchesToBeUpToDate?: boolean; + requiredConversationResolution?: boolean; sourcePath?: string; token?: string; }>({ @@ -76,6 +77,8 @@ export function createGithubRepoPushAction(options: { requiredStatusCheckContexts: inputProps.requiredStatusCheckContexts, bypassPullRequestAllowances: inputProps.bypassPullRequestAllowances, requireBranchesToBeUpToDate: inputProps.requireBranchesToBeUpToDate, + requiredConversationResolution: + inputProps.requiredConversationResolution, defaultBranch: inputProps.defaultBranch, protectDefaultBranch: inputProps.protectDefaultBranch, protectEnforceAdmins: inputProps.protectEnforceAdmins, @@ -108,6 +111,7 @@ export function createGithubRepoPushAction(options: { bypassPullRequestAllowances, requiredStatusCheckContexts = [], requireBranchesToBeUpToDate = true, + requiredConversationResolution = false, token: providedToken, } = ctx.input; @@ -146,6 +150,7 @@ export function createGithubRepoPushAction(options: { bypassPullRequestAllowances, requiredStatusCheckContexts, requireBranchesToBeUpToDate, + requiredConversationResolution, config, ctx.logger, gitCommitMessage, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/helpers.ts index 4dc7034f18..216a94ac57 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/helpers.ts @@ -258,6 +258,7 @@ export async function initRepoPushAndProtect( | undefined, requiredStatusCheckContexts: string[], requireBranchesToBeUpToDate: boolean, + requiredConversationResolution: boolean, config: Config, logger: any, gitCommitMessage?: string, @@ -303,6 +304,7 @@ export async function initRepoPushAndProtect( requireCodeOwnerReviews, requiredStatusCheckContexts, requireBranchesToBeUpToDate, + requiredConversationResolution, enforceAdmins: protectEnforceAdmins, dismissStaleReviews: dismissStaleReviews, }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/inputProperties.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/inputProperties.ts index 6c3bebf47e..a99a3030ac 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/inputProperties.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/inputProperties.ts @@ -58,6 +58,12 @@ const requireBranchesToBeUpToDate = { description: `Require branches to be up to date before merging. The default value is 'true'`, type: 'boolean', }; +const requiredConversationResolution = { + title: 'Required Conversation Resolution', + description: + 'Requires all conversations on code to be resolved before a pull request can be merged into this branch', + type: 'boolean', +}; const repoVisibility = { title: 'Repository Visibility', type: 'string', @@ -216,6 +222,7 @@ export { requireCodeOwnerReviews }; export { dismissStaleReviews }; export { requiredStatusCheckContexts }; export { requireBranchesToBeUpToDate }; +export { requiredConversationResolution }; export { sourcePath }; export { token }; export { topics }; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts index a7d29ec756..74e5d4580b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts @@ -191,6 +191,7 @@ type BranchProtectionOptions = { apps?: string[]; }; requireBranchesToBeUpToDate?: boolean; + requiredConversationResolution?: boolean; defaultBranch?: string; enforceAdmins?: boolean; dismissStaleReviews?: boolean; @@ -205,6 +206,7 @@ export const enableBranchProtectionOnDefaultRepoBranch = async ({ bypassPullRequestAllowances, requiredStatusCheckContexts = [], requireBranchesToBeUpToDate = true, + requiredConversationResolution = false, defaultBranch = 'master', enforceAdmins = true, dismissStaleReviews = false, @@ -237,6 +239,7 @@ export const enableBranchProtectionOnDefaultRepoBranch = async ({ bypass_pull_request_allowances: bypassPullRequestAllowances, dismiss_stale_reviews: dismissStaleReviews, }, + required_conversation_resolution: requiredConversationResolution, }); } catch (e) { assertError(e); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts index b5bda4c5d0..99a8753452 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts @@ -670,6 +670,7 @@ describe('publish:github', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -691,6 +692,7 @@ describe('publish:github', () => { requireCodeOwnerReviews: true, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -712,6 +714,7 @@ describe('publish:github', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -739,6 +742,7 @@ describe('publish:github', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -760,6 +764,7 @@ describe('publish:github', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: false, dismissStaleReviews: false, }); @@ -781,6 +786,7 @@ describe('publish:github', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -808,6 +814,7 @@ describe('publish:github', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -818,6 +825,7 @@ describe('publish:github', () => { ...mockContext.input, requiredStatusCheckContexts: ['statusCheck'], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, }, }); @@ -830,6 +838,7 @@ describe('publish:github', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: ['statusCheck'], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -852,6 +861,7 @@ describe('publish:github', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: ['statusCheck'], requireBranchesToBeUpToDate: false, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -873,6 +883,7 @@ describe('publish:github', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -955,6 +966,7 @@ describe('publish:github', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); @@ -976,6 +988,7 @@ describe('publish:github', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: true, }); @@ -997,6 +1010,7 @@ describe('publish:github', () => { requireCodeOwnerReviews: false, requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, enforceAdmins: true, dismissStaleReviews: false, }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index 1d3cce8067..e246234d2c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -71,6 +71,7 @@ export function createPublishGithubAction(options: { dismissStaleReviews?: boolean; requiredStatusCheckContexts?: string[]; requireBranchesToBeUpToDate?: boolean; + requiredConversationResolution?: boolean; repoVisibility?: 'private' | 'internal' | 'public'; collaborators?: Array< | { @@ -107,6 +108,8 @@ export function createPublishGithubAction(options: { dismissStaleReviews: inputProps.dismissStaleReviews, requiredStatusCheckContexts: inputProps.requiredStatusCheckContexts, requireBranchesToBeUpToDate: inputProps.requireBranchesToBeUpToDate, + requiredConversationResolution: + inputProps.requiredConversationResolution, repoVisibility: inputProps.repoVisibility, defaultBranch: inputProps.defaultBranch, protectDefaultBranch: inputProps.protectDefaultBranch, @@ -144,6 +147,7 @@ export function createPublishGithubAction(options: { bypassPullRequestAllowances, requiredStatusCheckContexts = [], requireBranchesToBeUpToDate = true, + requiredConversationResolution = false, repoVisibility = 'private', defaultBranch = 'master', protectDefaultBranch = true, @@ -211,6 +215,7 @@ export function createPublishGithubAction(options: { bypassPullRequestAllowances, requiredStatusCheckContexts, requireBranchesToBeUpToDate, + requiredConversationResolution, config, ctx.logger, gitCommitMessage, From a20a0ea69829695434d028c2d7916af281b5b125 Mon Sep 17 00:00:00 2001 From: Kyle Leonhard Date: Tue, 13 Dec 2022 15:55:07 -0800 Subject: [PATCH 073/237] Add changeset Signed-off-by: Kyle Leonhard --- .changeset/healthy-shrimps-notice.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/healthy-shrimps-notice.md diff --git a/.changeset/healthy-shrimps-notice.md b/.changeset/healthy-shrimps-notice.md new file mode 100644 index 0000000000..cca15de504 --- /dev/null +++ b/.changeset/healthy-shrimps-notice.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': minor +--- + +Add option to require Github conversation resolution From 71034ff1bffc6254c457f398a5e4a383e4b1207a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 14 Dec 2022 00:37:17 +0000 Subject: [PATCH 074/237] Update dependency @roadiehq/backstage-plugin-github-insights to v2.2.1 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 900facf289..8ab08ddd42 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12779,8 +12779,8 @@ __metadata: linkType: hard "@roadiehq/backstage-plugin-github-insights@npm:^2.0.5": - version: 2.2.0 - resolution: "@roadiehq/backstage-plugin-github-insights@npm:2.2.0" + version: 2.2.1 + resolution: "@roadiehq/backstage-plugin-github-insights@npm:2.2.1" dependencies: "@backstage/catalog-model": ^1.1.3 "@backstage/core-components": ^0.12.0 @@ -12802,7 +12802,7 @@ __metadata: react: ^16.13.1 || ^17.0.0 react-dom: ^16.13.1 || ^17.0.0 react-router: 6.0.0-beta.0 || ^6.3.0 - checksum: f48121c18cd15a0f1aa33991561ff06a438351de9aeadba810d08c95d129a7afc56341bf46aec87f2224c96ffdc810b176160b1aa416c698dba09204c626ddde + checksum: 4ef3c8d7da22a2cd1c5015e9e4d0782c60e5842f92fb072cc2b61f441fcfd10aa5ca6bda3e489600ac425ae9f9b730f88fa171f91ede00fdcd556a916c6d0d3a languageName: node linkType: hard From 25639f931a90cad46399fd97fa891847d8b64401 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 14 Dec 2022 01:25:39 +0000 Subject: [PATCH 075/237] Update dependency json-schema-library to v7.4.4 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 58398dc335..84c7545c84 100644 --- a/yarn.lock +++ b/yarn.lock @@ -26946,15 +26946,15 @@ __metadata: linkType: hard "json-schema-library@npm:^7.3.9": - version: 7.4.3 - resolution: "json-schema-library@npm:7.4.3" + version: 7.4.4 + resolution: "json-schema-library@npm:7.4.4" dependencies: "@sagold/json-pointer": ^5.0.0 "@sagold/json-query": ^6.0.0 deepmerge: ^4.2.2 fast-deep-equal: ^3.1.3 valid-url: ^1.0.9 - checksum: cc0d2a9e288fc6f1d56f0bd51dea74c978c931922c333e2c0a056ae3db0f20323ce44d304bcfd17afebc33c151c2c2c23fabe6ae1fc148599fa2c4e1d599a6c0 + checksum: e485c7f40bcfe04b96f457e02bc4ecb5bfe6b1d91df6a788909d1048a48bd51ecc5fe6bdd8759289a95c37fa63b3347b29d6d93f1b26a47f40cff21981d331d5 languageName: node linkType: hard From 07342778df3f4b4484f72ef808541d9c97fc6140 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Dec 2022 11:27:12 +0100 Subject: [PATCH 076/237] backend-app-api,backend-test-api: hook up shutdown hook to stop method Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- packages/backend-app-api/api-report.md | 2 ++ .../src/wiring/BackendInitializer.ts | 22 ++++++++++++ .../src/wiring/BackstageBackend.ts | 6 ++-- packages/backend-app-api/src/wiring/types.ts | 1 + packages/backend-test-utils/api-report.md | 3 +- .../src/next/wiring/TestBackend.test.ts | 35 +++++++++++++++++-- .../src/next/wiring/TestBackend.ts | 28 ++++++++++++--- 7 files changed, 87 insertions(+), 10 deletions(-) diff --git a/packages/backend-app-api/api-report.md b/packages/backend-app-api/api-report.md index 5008b1efe1..6cbb992ef8 100644 --- a/packages/backend-app-api/api-report.md +++ b/packages/backend-app-api/api-report.md @@ -26,6 +26,8 @@ export interface Backend { add(feature: BackendFeature): void; // (undocumented) start(): Promise; + // (undocumented) + stop(): Promise; } // @public (undocumented) diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.ts b/packages/backend-app-api/src/wiring/BackendInitializer.ts index c86cd8382d..5a5f65a5a9 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.ts @@ -17,8 +17,10 @@ import { BackendFeature, ExtensionPoint, + lifecycleServiceRef, ServiceRef, } from '@backstage/backend-plugin-api'; +import { BackendLifecycleImpl } from '../services/implementations/lifecycleService'; import { BackendRegisterInit, ServiceHolder, @@ -165,4 +167,24 @@ export class BackendInitializer { return orderedRegisterInits; } + + async stop(): Promise { + if (!this.#started) { + throw new Error('Backend has not started'); + } + this.#started = false; + + const lifecycleService = await this.#serviceHolder.get( + lifecycleServiceRef, + 'root', + ); + + // TODO(Rugvip): :D + const lifecycle = (lifecycleService as any)?.lifecycle; + if (lifecycle instanceof BackendLifecycleImpl) { + await lifecycle.shutdown(); + } else { + throw new Error('Unexpected lifecycle service implementation'); + } + } } diff --git a/packages/backend-app-api/src/wiring/BackstageBackend.ts b/packages/backend-app-api/src/wiring/BackstageBackend.ts index 1d76161bfe..c53f47b685 100644 --- a/packages/backend-app-api/src/wiring/BackstageBackend.ts +++ b/packages/backend-app-api/src/wiring/BackstageBackend.ts @@ -36,7 +36,7 @@ export class BackstageBackend implements Backend { await this.#initializer.start(); } - // async stop(): Promise { - // await this.#initializer.stop(); - // } + async stop(): Promise { + await this.#initializer.stop(); + } } diff --git a/packages/backend-app-api/src/wiring/types.ts b/packages/backend-app-api/src/wiring/types.ts index 40ce6aa1ae..3e1acce08e 100644 --- a/packages/backend-app-api/src/wiring/types.ts +++ b/packages/backend-app-api/src/wiring/types.ts @@ -28,6 +28,7 @@ import { BackstageBackend } from './BackstageBackend'; export interface Backend { add(feature: BackendFeature): void; start(): Promise; + stop(): Promise; } export interface BackendRegisterInit { diff --git a/packages/backend-test-utils/api-report.md b/packages/backend-test-utils/api-report.md index 2fe4ed31d3..3df3f3cb21 100644 --- a/packages/backend-test-utils/api-report.md +++ b/packages/backend-test-utils/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { Backend } from '@backstage/backend-app-api'; import { BackendFeature } from '@backstage/backend-plugin-api'; import { ExtensionPoint } from '@backstage/backend-plugin-api'; import { Knex } from 'knex'; @@ -23,7 +24,7 @@ export function setupRequestMockHandlers(worker: { export function startTestBackend< TServices extends any[], TExtensionPoints extends any[], ->(options: TestBackendOptions): Promise; +>(options: TestBackendOptions): Promise; // @alpha (undocumented) export interface TestBackendOptions< diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts index 164e06d3b5..b87e157477 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts @@ -19,6 +19,7 @@ import { createExtensionPoint, createServiceFactory, createServiceRef, + lifecycleServiceRef, } from '@backstage/backend-plugin-api'; import { startTestBackend } from './TestBackend'; @@ -31,7 +32,7 @@ describe('TestBackend', () => { const extensionPoint3 = createExtensionPoint({ id: 'b3' }); const extensionPoint4 = createExtensionPoint({ id: 'b4' }); const extensionPoint5 = createExtensionPoint({ id: 'b5' }); - await startTestBackend({ + const backend = await startTestBackend({ services: [ // @ts-expect-error [extensionPoint1, { a: 'a' }], @@ -58,6 +59,7 @@ describe('TestBackend', () => { ], }); expect(1).toBe(1); + await backend.stop(); }); it('should start the test backend', async () => { @@ -87,11 +89,40 @@ describe('TestBackend', () => { }, }); - await startTestBackend({ + const backend = await startTestBackend({ services: [sf], features: [testModule()], }); expect(testFn).toHaveBeenCalledWith('winning'); + await backend.stop(); + }); + + it('should stop the test backend', async () => { + const shutdownSpy = jest.fn(); + + const testModule = createBackendModule({ + moduleId: 'test.module', + pluginId: 'test', + register(env) { + env.registerInit({ + deps: { + lifecycle: lifecycleServiceRef, + }, + async init({ lifecycle }) { + lifecycle.addShutdownHook({ fn: shutdownSpy }); + }, + }); + }, + }); + + const backend = await startTestBackend({ + services: [], + features: [testModule()], + }); + + expect(shutdownSpy).not.toHaveBeenCalled(); + await backend.stop(); + expect(shutdownSpy).toHaveBeenCalled(); }); }); diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.ts index 677ba930cc..9c9427e332 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.ts @@ -14,7 +14,13 @@ * limitations under the License. */ -import { createSpecializedBackend } from '@backstage/backend-app-api'; +import { + Backend, + createSpecializedBackend, + lifecycleFactory, + loggerFactory, + rootLoggerFactory, +} from '@backstage/backend-app-api'; import { ServiceFactory, ServiceRef, @@ -47,11 +53,17 @@ export interface TestBackendOptions< features?: BackendFeature[]; } +const defaultServiceFactories = [ + rootLoggerFactory(), + loggerFactory(), + lifecycleFactory(), +]; + /** @alpha */ export async function startTestBackend< TServices extends any[], TExtensionPoints extends any[], ->(options: TestBackendOptions): Promise { +>(options: TestBackendOptions): Promise { const { services = [], extensionPoints = [], @@ -69,17 +81,23 @@ export async function startTestBackend< service: ref, deps: {}, factory: async () => async () => impl, - }); + })(); } return createServiceFactory({ service: ref, deps: {}, factory: async () => impl, - }); + })(); } return serviceDef as ServiceFactory; }); + for (const factory of defaultServiceFactories) { + if (!factories.some(f => f.service === factory.service)) { + factories.push(factory); + } + } + const backend = createSpecializedBackend({ ...otherOptions, services: factories, @@ -101,4 +119,6 @@ export async function startTestBackend< } await backend.start(); + + return backend; } From 52a9bd35c7b3f87e6038a01b0968335709090454 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Dec 2022 14:01:28 +0100 Subject: [PATCH 077/237] backend-test-utils: automatically stop backends after test Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- .../src/wiring/BackendInitializer.ts | 2 +- packages/backend-test-utils/api-report.md | 2 ++ .../src/next/wiring/TestBackend.test.ts | 7 ++-- .../src/next/wiring/TestBackend.ts | 32 +++++++++++++++++++ 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.ts b/packages/backend-app-api/src/wiring/BackendInitializer.ts index 5a5f65a5a9..c3e2a3fa3d 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.ts @@ -170,7 +170,7 @@ export class BackendInitializer { async stop(): Promise { if (!this.#started) { - throw new Error('Backend has not started'); + return; } this.#started = false; diff --git a/packages/backend-test-utils/api-report.md b/packages/backend-test-utils/api-report.md index 3df3f3cb21..90a315f745 100644 --- a/packages/backend-test-utils/api-report.md +++ b/packages/backend-test-utils/api-report.md @@ -31,6 +31,8 @@ export interface TestBackendOptions< TServices extends any[], TExtensionPoints extends any[], > { + // (undocumented) + autoStop?: boolean; // (undocumented) extensionPoints?: readonly [ ...{ diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts index b87e157477..383a182db2 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts @@ -32,7 +32,7 @@ describe('TestBackend', () => { const extensionPoint3 = createExtensionPoint({ id: 'b3' }); const extensionPoint4 = createExtensionPoint({ id: 'b4' }); const extensionPoint5 = createExtensionPoint({ id: 'b5' }); - const backend = await startTestBackend({ + await startTestBackend({ services: [ // @ts-expect-error [extensionPoint1, { a: 'a' }], @@ -59,7 +59,6 @@ describe('TestBackend', () => { ], }); expect(1).toBe(1); - await backend.stop(); }); it('should start the test backend', async () => { @@ -89,13 +88,12 @@ describe('TestBackend', () => { }, }); - const backend = await startTestBackend({ + await startTestBackend({ services: [sf], features: [testModule()], }); expect(testFn).toHaveBeenCalledWith('winning'); - await backend.stop(); }); it('should stop the test backend', async () => { @@ -119,6 +117,7 @@ describe('TestBackend', () => { const backend = await startTestBackend({ services: [], features: [testModule()], + autoStop: false, }); expect(shutdownSpy).not.toHaveBeenCalled(); diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.ts index 9c9427e332..dbb4cff59a 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.ts @@ -51,6 +51,7 @@ export interface TestBackendOptions< }, ]; features?: BackendFeature[]; + autoStop?: boolean; } const defaultServiceFactories = [ @@ -59,6 +60,8 @@ const defaultServiceFactories = [ lifecycleFactory(), ]; +const backendInstances = new Array(); + /** @alpha */ export async function startTestBackend< TServices extends any[], @@ -68,6 +71,7 @@ export async function startTestBackend< services = [], extensionPoints = [], features = [], + autoStop = true, ...otherOptions } = options; @@ -103,6 +107,10 @@ export async function startTestBackend< services: factories, }); + if (autoStop) { + backendInstances.push(backend); + } + backend.add({ id: `---test-extension-point-registrar`, register(reg) { @@ -122,3 +130,27 @@ export async function startTestBackend< return backend; } + +let registered = false; +function registerTestHooks() { + if (typeof afterEach !== 'function') { + return; + } + if (registered) { + return; + } + registered = true; + + afterEach(async () => { + for (const backend of backendInstances) { + try { + await backend.stop(); + } catch (error) { + console.error(`Failed to stop backend after test, ${error}`); + } + } + backendInstances.length = 0; + }); +} + +registerTestHooks(); From afa3bf5657745deed1518bd38c7d4b4cf39ad7de Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Dec 2022 14:03:24 +0100 Subject: [PATCH 078/237] changesets: added changeset for backend-test-utils auto stop Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- .changeset/green-moose-leave.md | 5 +++++ .changeset/nasty-colts-cough.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/green-moose-leave.md create mode 100644 .changeset/nasty-colts-cough.md diff --git a/.changeset/green-moose-leave.md b/.changeset/green-moose-leave.md new file mode 100644 index 0000000000..23e4b6878f --- /dev/null +++ b/.changeset/green-moose-leave.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-test-utils': patch +--- + +Backends started with `startTestBackend` are now automatically stopped after the each test, unless the `autoStop` option is set to `false`. diff --git a/.changeset/nasty-colts-cough.md b/.changeset/nasty-colts-cough.md new file mode 100644 index 0000000000..5dd098ec27 --- /dev/null +++ b/.changeset/nasty-colts-cough.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-app-api': patch +--- + +Added `.stop()` method to `Backend`. From 1f2edda7c3b5663fff4995b53e4dbfb41296e804 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Dec 2022 16:37:50 +0100 Subject: [PATCH 079/237] backend-app-api: text Signed-off-by: Patrik Oldsberg --- packages/backend-app-api/src/wiring/BackendInitializer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.ts b/packages/backend-app-api/src/wiring/BackendInitializer.ts index c3e2a3fa3d..dca4a08b8e 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.ts @@ -179,7 +179,7 @@ export class BackendInitializer { 'root', ); - // TODO(Rugvip): :D + // TODO(Rugvip): Find a better way to do this const lifecycle = (lifecycleService as any)?.lifecycle; if (lifecycle instanceof BackendLifecycleImpl) { await lifecycle.shutdown(); From e507d9dbb76747cc23e7d099f3f0f88b8c75a66b Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Dec 2022 16:41:00 +0100 Subject: [PATCH 080/237] backend-test-utils: switch autoStop option to use afterEach | never Signed-off-by: Patrik Oldsberg --- packages/backend-test-utils/api-report.md | 2 +- .../backend-test-utils/src/next/wiring/TestBackend.test.ts | 2 +- packages/backend-test-utils/src/next/wiring/TestBackend.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend-test-utils/api-report.md b/packages/backend-test-utils/api-report.md index 90a315f745..b791f90726 100644 --- a/packages/backend-test-utils/api-report.md +++ b/packages/backend-test-utils/api-report.md @@ -32,7 +32,7 @@ export interface TestBackendOptions< TExtensionPoints extends any[], > { // (undocumented) - autoStop?: boolean; + autoStop?: 'afterEach' | 'never'; // (undocumented) extensionPoints?: readonly [ ...{ diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts index 383a182db2..21200fe281 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts @@ -117,7 +117,7 @@ describe('TestBackend', () => { const backend = await startTestBackend({ services: [], features: [testModule()], - autoStop: false, + autoStop: 'never', }); expect(shutdownSpy).not.toHaveBeenCalled(); diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.ts index dbb4cff59a..a273e879ce 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.ts @@ -51,7 +51,7 @@ export interface TestBackendOptions< }, ]; features?: BackendFeature[]; - autoStop?: boolean; + autoStop?: 'afterEach' | 'never'; } const defaultServiceFactories = [ @@ -71,7 +71,7 @@ export async function startTestBackend< services = [], extensionPoints = [], features = [], - autoStop = true, + autoStop = 'afterEach', ...otherOptions } = options; From bb41a58c6abb1962c727e63aebf399ee4ee37f16 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 2 Dec 2022 16:43:54 +0100 Subject: [PATCH 081/237] fix coreServices conflicts Signed-off-by: Patrik Oldsberg --- packages/backend-app-api/src/wiring/BackendInitializer.ts | 4 ++-- .../backend-test-utils/src/next/wiring/TestBackend.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.ts b/packages/backend-app-api/src/wiring/BackendInitializer.ts index dca4a08b8e..c423096d36 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.ts @@ -17,7 +17,7 @@ import { BackendFeature, ExtensionPoint, - lifecycleServiceRef, + coreServices, ServiceRef, } from '@backstage/backend-plugin-api'; import { BackendLifecycleImpl } from '../services/implementations/lifecycleService'; @@ -175,7 +175,7 @@ export class BackendInitializer { this.#started = false; const lifecycleService = await this.#serviceHolder.get( - lifecycleServiceRef, + coreServices.lifecycle, 'root', ); diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts index 21200fe281..417dfa0477 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts @@ -19,7 +19,7 @@ import { createExtensionPoint, createServiceFactory, createServiceRef, - lifecycleServiceRef, + coreServices, } from '@backstage/backend-plugin-api'; import { startTestBackend } from './TestBackend'; @@ -105,7 +105,7 @@ describe('TestBackend', () => { register(env) { env.registerInit({ deps: { - lifecycle: lifecycleServiceRef, + lifecycle: coreServices.lifecycle, }, async init({ lifecycle }) { lifecycle.addShutdownHook({ fn: shutdownSpy }); From deaa069b76a69da21e9c4067be33f1da69373e1c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 14 Dec 2022 11:27:19 +0100 Subject: [PATCH 082/237] backend-app-api: do not allow backends to be restarted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- packages/backend-app-api/src/wiring/BackendInitializer.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.ts b/packages/backend-app-api/src/wiring/BackendInitializer.ts index c423096d36..c8da028f26 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.ts @@ -172,7 +172,6 @@ export class BackendInitializer { if (!this.#started) { return; } - this.#started = false; const lifecycleService = await this.#serviceHolder.get( coreServices.lifecycle, From 98ba60b8b580e1c88ed458c111de88cc6985b220 Mon Sep 17 00:00:00 2001 From: Gabriel Testault Date: Wed, 14 Dec 2022 11:29:44 +0100 Subject: [PATCH 083/237] feature(jenkins-backend): improve test for standalone project test that the client is called twice for unfiltered standalone projects. Signed-off-by: Gabriel Testault --- plugins/jenkins-backend/src/service/jenkinsApi.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/jenkins-backend/src/service/jenkinsApi.test.ts b/plugins/jenkins-backend/src/service/jenkinsApi.test.ts index 8413cc6699..57ce2dbed4 100644 --- a/plugins/jenkins-backend/src/service/jenkinsApi.test.ts +++ b/plugins/jenkins-backend/src/service/jenkinsApi.test.ts @@ -81,6 +81,7 @@ describe('JenkinsApi', () => { .mockResolvedValueOnce(project) .mockResolvedValueOnce(project); const result = await jenkinsApi.getProjects(jenkinsInfo); + expect(mockedJenkinsClient.job.get).toHaveBeenCalledTimes(2); expect(result).toHaveLength(1); expect(result[0]).toEqual({ actions: [], From d657e194245bbf85f0724e8c6e43408ff40066a0 Mon Sep 17 00:00:00 2001 From: Gabriel Testault Date: Wed, 14 Dec 2022 11:30:40 +0100 Subject: [PATCH 084/237] feature(jenkins-backend): downgrade change to patch level Signed-off-by: Gabriel Testault --- .changeset/seven-dolphins-tickle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/seven-dolphins-tickle.md b/.changeset/seven-dolphins-tickle.md index b3b7697dd6..cd3149792c 100644 --- a/.changeset/seven-dolphins-tickle.md +++ b/.changeset/seven-dolphins-tickle.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-jenkins-backend': minor +'@backstage/plugin-jenkins-backend': patch --- added support for standalone jenkins projects From 6c70d769e4af276103bfa805eb9a8b1ca918a22c Mon Sep 17 00:00:00 2001 From: Gabriel Testault Date: Wed, 14 Dec 2022 12:02:27 +0100 Subject: [PATCH 085/237] feature(jenkins-backend): be more precise in jenkins terminology Only explicitly support Multibranch Pipeline and Pipeline projects Signed-off-by: Gabriel Testault --- plugins/jenkins-backend/src/service/jenkinsApi.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/plugins/jenkins-backend/src/service/jenkinsApi.ts b/plugins/jenkins-backend/src/service/jenkinsApi.ts index be8fe1b983..ef4ddc47b0 100644 --- a/plugins/jenkins-backend/src/service/jenkinsApi.ts +++ b/plugins/jenkins-backend/src/service/jenkinsApi.ts @@ -75,7 +75,7 @@ export class JenkinsApiImpl { const projects: BackstageProject[] = []; if (branches) { - // Assume jenkinsInfo.jobFullName is a folder which contains one job per branch. + // Assume jenkinsInfo.jobFullName is a MultiBranch Pipeline project which contains one job per branch. // TODO: extract a strategy interface for this const job = await Promise.any( branches.map(branch => @@ -88,7 +88,9 @@ export class JenkinsApiImpl { projects.push(this.augmentProject(job)); } else { // We aren't filtering - // Assume jenkinsInfo.jobFullName is a folder which contains one job per branch. + // Assume jenkinsInfo.jobFullName is either + // a MultiBranch Pipeline (folder with one job per branch) project + // a Pipeline (standalone) project const project = await client.job.get({ name: jenkinsInfo.jobFullName, // Filter only be the information we need, instead of loading all fields. @@ -110,12 +112,7 @@ export class JenkinsApiImpl { } for (const jobDetails of project.jobs) { // for each branch (we assume) - if (jobDetails?.jobs) { - // skipping folders inside folders for now - // TODO: recurse - } else { - projects.push(this.augmentProject(jobDetails)); - } + projects.push(this.augmentProject(jobDetails)); } } return projects; From 73a8f69d62076750efc8a2fd59292b2d9300e5e8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 14 Dec 2022 11:27:43 +0100 Subject: [PATCH 086/237] backend-test-utils: only clean up test backends after all tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- packages/backend-test-utils/api-report.md | 2 -- .../src/next/wiring/TestBackend.test.ts | 33 ++++++++++++++++++- .../src/next/wiring/TestBackend.ts | 16 ++++----- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/packages/backend-test-utils/api-report.md b/packages/backend-test-utils/api-report.md index b791f90726..3df3f3cb21 100644 --- a/packages/backend-test-utils/api-report.md +++ b/packages/backend-test-utils/api-report.md @@ -31,8 +31,6 @@ export interface TestBackendOptions< TServices extends any[], TExtensionPoints extends any[], > { - // (undocumented) - autoStop?: 'afterEach' | 'never'; // (undocumented) extensionPoints?: readonly [ ...{ diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts index 417dfa0477..3cc8507df3 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.test.ts @@ -23,6 +23,38 @@ import { } from '@backstage/backend-plugin-api'; import { startTestBackend } from './TestBackend'; +// This bit makes sure that test backends are cleaned up properly +let globalTestBackendHasBeenStopped = false; +beforeAll(async () => { + await startTestBackend({ + services: [], + features: [ + createBackendModule({ + moduleId: 'test.module', + pluginId: 'test', + register(env) { + env.registerInit({ + deps: { lifecycle: coreServices.lifecycle }, + async init({ lifecycle }) { + lifecycle.addShutdownHook({ + fn() { + globalTestBackendHasBeenStopped = true; + }, + }); + }, + }); + }, + })(), + ], + }); +}); + +afterAll(() => { + if (!globalTestBackendHasBeenStopped) { + throw new Error('Expected backend to have been stopped'); + } +}); + describe('TestBackend', () => { it('should get a type error if service implementation does not match', async () => { type Obj = { a: string; b: string }; @@ -117,7 +149,6 @@ describe('TestBackend', () => { const backend = await startTestBackend({ services: [], features: [testModule()], - autoStop: 'never', }); expect(shutdownSpy).not.toHaveBeenCalled(); diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.ts index a273e879ce..7dbcf439dc 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.ts @@ -51,7 +51,6 @@ export interface TestBackendOptions< }, ]; features?: BackendFeature[]; - autoStop?: 'afterEach' | 'never'; } const defaultServiceFactories = [ @@ -60,7 +59,7 @@ const defaultServiceFactories = [ lifecycleFactory(), ]; -const backendInstances = new Array(); +const backendInstancesToCleanUp = new Array(); /** @alpha */ export async function startTestBackend< @@ -71,7 +70,6 @@ export async function startTestBackend< services = [], extensionPoints = [], features = [], - autoStop = 'afterEach', ...otherOptions } = options; @@ -107,9 +105,7 @@ export async function startTestBackend< services: factories, }); - if (autoStop) { - backendInstances.push(backend); - } + backendInstancesToCleanUp.push(backend); backend.add({ id: `---test-extension-point-registrar`, @@ -141,15 +137,15 @@ function registerTestHooks() { } registered = true; - afterEach(async () => { - for (const backend of backendInstances) { + afterAll(async () => { + for (const backend of backendInstancesToCleanUp) { try { await backend.stop(); } catch (error) { - console.error(`Failed to stop backend after test, ${error}`); + console.error(`Failed to stop backend after tests, ${error}`); } } - backendInstances.length = 0; + backendInstancesToCleanUp.length = 0; }); } From cdc5ac632022584084485b520a2804f8a3aeb9f3 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 14 Dec 2022 13:23:45 +0000 Subject: [PATCH 087/237] Update dependency esbuild to v0.16.6 Signed-off-by: Renovate Bot --- yarn.lock | 182 +++++++++++++++++++++++++++--------------------------- 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/yarn.lock b/yarn.lock index 84c7545c84..1cda4df54c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9124,9 +9124,9 @@ __metadata: languageName: node linkType: hard -"@esbuild/android-arm64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/android-arm64@npm:0.16.4" +"@esbuild/android-arm64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/android-arm64@npm:0.16.6" conditions: os=android & cpu=arm64 languageName: node linkType: hard @@ -9138,65 +9138,65 @@ __metadata: languageName: node linkType: hard -"@esbuild/android-arm@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/android-arm@npm:0.16.4" +"@esbuild/android-arm@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/android-arm@npm:0.16.6" conditions: os=android & cpu=arm languageName: node linkType: hard -"@esbuild/android-x64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/android-x64@npm:0.16.4" +"@esbuild/android-x64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/android-x64@npm:0.16.6" conditions: os=android & cpu=x64 languageName: node linkType: hard -"@esbuild/darwin-arm64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/darwin-arm64@npm:0.16.4" +"@esbuild/darwin-arm64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/darwin-arm64@npm:0.16.6" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@esbuild/darwin-x64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/darwin-x64@npm:0.16.4" +"@esbuild/darwin-x64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/darwin-x64@npm:0.16.6" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@esbuild/freebsd-arm64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/freebsd-arm64@npm:0.16.4" +"@esbuild/freebsd-arm64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/freebsd-arm64@npm:0.16.6" conditions: os=freebsd & cpu=arm64 languageName: node linkType: hard -"@esbuild/freebsd-x64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/freebsd-x64@npm:0.16.4" +"@esbuild/freebsd-x64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/freebsd-x64@npm:0.16.6" conditions: os=freebsd & cpu=x64 languageName: node linkType: hard -"@esbuild/linux-arm64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/linux-arm64@npm:0.16.4" +"@esbuild/linux-arm64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/linux-arm64@npm:0.16.6" conditions: os=linux & cpu=arm64 languageName: node linkType: hard -"@esbuild/linux-arm@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/linux-arm@npm:0.16.4" +"@esbuild/linux-arm@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/linux-arm@npm:0.16.6" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@esbuild/linux-ia32@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/linux-ia32@npm:0.16.4" +"@esbuild/linux-ia32@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/linux-ia32@npm:0.16.6" conditions: os=linux & cpu=ia32 languageName: node linkType: hard @@ -9208,86 +9208,86 @@ __metadata: languageName: node linkType: hard -"@esbuild/linux-loong64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/linux-loong64@npm:0.16.4" +"@esbuild/linux-loong64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/linux-loong64@npm:0.16.6" conditions: os=linux & cpu=loong64 languageName: node linkType: hard -"@esbuild/linux-mips64el@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/linux-mips64el@npm:0.16.4" +"@esbuild/linux-mips64el@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/linux-mips64el@npm:0.16.6" conditions: os=linux & cpu=mips64el languageName: node linkType: hard -"@esbuild/linux-ppc64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/linux-ppc64@npm:0.16.4" +"@esbuild/linux-ppc64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/linux-ppc64@npm:0.16.6" conditions: os=linux & cpu=ppc64 languageName: node linkType: hard -"@esbuild/linux-riscv64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/linux-riscv64@npm:0.16.4" +"@esbuild/linux-riscv64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/linux-riscv64@npm:0.16.6" conditions: os=linux & cpu=riscv64 languageName: node linkType: hard -"@esbuild/linux-s390x@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/linux-s390x@npm:0.16.4" +"@esbuild/linux-s390x@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/linux-s390x@npm:0.16.6" conditions: os=linux & cpu=s390x languageName: node linkType: hard -"@esbuild/linux-x64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/linux-x64@npm:0.16.4" +"@esbuild/linux-x64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/linux-x64@npm:0.16.6" conditions: os=linux & cpu=x64 languageName: node linkType: hard -"@esbuild/netbsd-x64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/netbsd-x64@npm:0.16.4" +"@esbuild/netbsd-x64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/netbsd-x64@npm:0.16.6" conditions: os=netbsd & cpu=x64 languageName: node linkType: hard -"@esbuild/openbsd-x64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/openbsd-x64@npm:0.16.4" +"@esbuild/openbsd-x64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/openbsd-x64@npm:0.16.6" conditions: os=openbsd & cpu=x64 languageName: node linkType: hard -"@esbuild/sunos-x64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/sunos-x64@npm:0.16.4" +"@esbuild/sunos-x64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/sunos-x64@npm:0.16.6" conditions: os=sunos & cpu=x64 languageName: node linkType: hard -"@esbuild/win32-arm64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/win32-arm64@npm:0.16.4" +"@esbuild/win32-arm64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/win32-arm64@npm:0.16.6" conditions: os=win32 & cpu=arm64 languageName: node linkType: hard -"@esbuild/win32-ia32@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/win32-ia32@npm:0.16.4" +"@esbuild/win32-ia32@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/win32-ia32@npm:0.16.6" conditions: os=win32 & cpu=ia32 languageName: node linkType: hard -"@esbuild/win32-x64@npm:0.16.4": - version: 0.16.4 - resolution: "@esbuild/win32-x64@npm:0.16.4" +"@esbuild/win32-x64@npm:0.16.6": + version: 0.16.6 + resolution: "@esbuild/win32-x64@npm:0.16.6" conditions: os=win32 & cpu=x64 languageName: node linkType: hard @@ -21526,31 +21526,31 @@ __metadata: linkType: hard "esbuild@npm:^0.16.0": - version: 0.16.4 - resolution: "esbuild@npm:0.16.4" + version: 0.16.6 + resolution: "esbuild@npm:0.16.6" dependencies: - "@esbuild/android-arm": 0.16.4 - "@esbuild/android-arm64": 0.16.4 - "@esbuild/android-x64": 0.16.4 - "@esbuild/darwin-arm64": 0.16.4 - "@esbuild/darwin-x64": 0.16.4 - "@esbuild/freebsd-arm64": 0.16.4 - "@esbuild/freebsd-x64": 0.16.4 - "@esbuild/linux-arm": 0.16.4 - "@esbuild/linux-arm64": 0.16.4 - "@esbuild/linux-ia32": 0.16.4 - "@esbuild/linux-loong64": 0.16.4 - "@esbuild/linux-mips64el": 0.16.4 - "@esbuild/linux-ppc64": 0.16.4 - "@esbuild/linux-riscv64": 0.16.4 - "@esbuild/linux-s390x": 0.16.4 - "@esbuild/linux-x64": 0.16.4 - "@esbuild/netbsd-x64": 0.16.4 - "@esbuild/openbsd-x64": 0.16.4 - "@esbuild/sunos-x64": 0.16.4 - "@esbuild/win32-arm64": 0.16.4 - "@esbuild/win32-ia32": 0.16.4 - "@esbuild/win32-x64": 0.16.4 + "@esbuild/android-arm": 0.16.6 + "@esbuild/android-arm64": 0.16.6 + "@esbuild/android-x64": 0.16.6 + "@esbuild/darwin-arm64": 0.16.6 + "@esbuild/darwin-x64": 0.16.6 + "@esbuild/freebsd-arm64": 0.16.6 + "@esbuild/freebsd-x64": 0.16.6 + "@esbuild/linux-arm": 0.16.6 + "@esbuild/linux-arm64": 0.16.6 + "@esbuild/linux-ia32": 0.16.6 + "@esbuild/linux-loong64": 0.16.6 + "@esbuild/linux-mips64el": 0.16.6 + "@esbuild/linux-ppc64": 0.16.6 + "@esbuild/linux-riscv64": 0.16.6 + "@esbuild/linux-s390x": 0.16.6 + "@esbuild/linux-x64": 0.16.6 + "@esbuild/netbsd-x64": 0.16.6 + "@esbuild/openbsd-x64": 0.16.6 + "@esbuild/sunos-x64": 0.16.6 + "@esbuild/win32-arm64": 0.16.6 + "@esbuild/win32-ia32": 0.16.6 + "@esbuild/win32-x64": 0.16.6 dependenciesMeta: "@esbuild/android-arm": optional: true @@ -21598,7 +21598,7 @@ __metadata: optional: true bin: esbuild: bin/esbuild - checksum: c06e9b2e84f5c7cdb608fa15e5a241d155321097fe1362beab176bc8283f54ae2a9a7fcca741da2663ffb5fea98c6c47226edd22189d3effb14b457e46592d1b + checksum: b77f9f516824f5a4e5670367b6cc5f3e649ac629f2d2b70811ff06c1cd6fb71941f470ed40e85c57f357f4e7a984540304f20b3c6090a0a4e12b4fe1850e2734 languageName: node linkType: hard From 5260d8fc7df93e1401dff1f55917cffa6346b303 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 14 Dec 2022 14:12:34 +0100 Subject: [PATCH 088/237] backend-app-api: always initialize root services MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- .changeset/shy-boats-admire.md | 5 ++ .../src/wiring/BackendInitializer.test.ts | 57 +++++++++++++++++++ .../src/wiring/BackendInitializer.ts | 14 ++++- .../src/wiring/ServiceRegistry.ts | 8 ++- packages/backend-app-api/src/wiring/types.ts | 11 +++- 5 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 .changeset/shy-boats-admire.md create mode 100644 packages/backend-app-api/src/wiring/BackendInitializer.test.ts diff --git a/.changeset/shy-boats-admire.md b/.changeset/shy-boats-admire.md new file mode 100644 index 0000000000..904fde9d1e --- /dev/null +++ b/.changeset/shy-boats-admire.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-app-api': patch +--- + +Root scoped services are now always initialized, regardless of whether they're used by any features. diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.test.ts b/packages/backend-app-api/src/wiring/BackendInitializer.test.ts new file mode 100644 index 0000000000..8fd7441c37 --- /dev/null +++ b/packages/backend-app-api/src/wiring/BackendInitializer.test.ts @@ -0,0 +1,57 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + createServiceRef, + createServiceFactory, +} from '@backstage/backend-plugin-api'; +import { BackendInitializer } from './BackendInitializer'; +import { ServiceRegistry } from './ServiceRegistry'; + +const rootRef = createServiceRef<{ x: number }>({ + id: '1', + scope: 'root', +}); + +const pluginRef = createServiceRef<{ x: number }>({ + id: '2', +}); + +describe('BackendInitializer', () => { + it('should initialize root scoped services', async () => { + const rootFactory = jest.fn(); + const pluginFactory = jest.fn(); + + const registry = new ServiceRegistry([ + createServiceFactory({ + service: rootRef, + deps: {}, + factory: rootFactory, + }), + createServiceFactory({ + service: pluginRef, + deps: {}, + factory: pluginFactory, + }), + ]); + + const init = new BackendInitializer(registry); + await init.start(); + + expect(rootFactory).toHaveBeenCalled(); + expect(pluginFactory).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/backend-app-api/src/wiring/BackendInitializer.ts b/packages/backend-app-api/src/wiring/BackendInitializer.ts index c86cd8382d..8311ae10bb 100644 --- a/packages/backend-app-api/src/wiring/BackendInitializer.ts +++ b/packages/backend-app-api/src/wiring/BackendInitializer.ts @@ -21,7 +21,7 @@ import { } from '@backstage/backend-plugin-api'; import { BackendRegisterInit, - ServiceHolder, + EnumerableServiceHolder, ServiceOrExtensionPoint, } from './types'; @@ -30,9 +30,9 @@ export class BackendInitializer { #features = new Map(); #registerInits = new Array(); #extensionPoints = new Map, unknown>(); - #serviceHolder: ServiceHolder; + #serviceHolder: EnumerableServiceHolder; - constructor(serviceHolder: ServiceHolder) { + constructor(serviceHolder: EnumerableServiceHolder) { this.#serviceHolder = serviceHolder; } @@ -85,6 +85,14 @@ export class BackendInitializer { } this.#started = true; + // Initialize all root scoped services + for (const ref of this.#serviceHolder.getServiceRefs()) { + if (ref.scope === 'root') { + await this.#serviceHolder.get(ref, 'root'); + } + } + + // Initialize all features for (const [feature] of this.#features) { const provides = new Set>(); diff --git a/packages/backend-app-api/src/wiring/ServiceRegistry.ts b/packages/backend-app-api/src/wiring/ServiceRegistry.ts index 9ff73f4270..208e5cf260 100644 --- a/packages/backend-app-api/src/wiring/ServiceRegistry.ts +++ b/packages/backend-app-api/src/wiring/ServiceRegistry.ts @@ -20,7 +20,7 @@ import { coreServices, } from '@backstage/backend-plugin-api'; import { stringifyError } from '@backstage/errors'; - +import { EnumerableServiceHolder } from './types'; /** * Keep in sync with `@backstage/backend-plugin-api/src/services/system/types.ts` * @internal @@ -31,7 +31,7 @@ export type InternalServiceRef = ServiceRef & { ) => Promise | (() => ServiceFactory)>; }; -export class ServiceRegistry { +export class ServiceRegistry implements EnumerableServiceHolder { readonly #providedFactories: Map; readonly #loadedDefaultFactories: Map>; readonly #implementations: Map< @@ -132,6 +132,10 @@ export class ServiceRegistry { } } + getServiceRefs(): ServiceRef[] { + return Array.from(this.#providedFactories.values()).map(f => f.service); + } + get(ref: ServiceRef, pluginId: string): Promise | undefined { return this.#resolveFactory(ref, pluginId)?.then(factory => { if (factory.scope === 'root') { diff --git a/packages/backend-app-api/src/wiring/types.ts b/packages/backend-app-api/src/wiring/types.ts index 40ce6aa1ae..76bcd5ab6d 100644 --- a/packages/backend-app-api/src/wiring/types.ts +++ b/packages/backend-app-api/src/wiring/types.ts @@ -45,9 +45,16 @@ export interface CreateSpecializedBackendOptions { services: (ServiceFactory | (() => ServiceFactory))[]; } -export type ServiceHolder = { +export interface ServiceHolder { get(api: ServiceRef, pluginId: string): Promise | undefined; -}; +} + +/** + * @internal + */ +export interface EnumerableServiceHolder extends ServiceHolder { + getServiceRefs(): ServiceRef[]; +} /** * @public From a0251905524ca010538957dfa4ab40b874a4f9f2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 14 Dec 2022 14:33:33 +0100 Subject: [PATCH 089/237] backend-plugin-api: suffix all service interfaces with *Service Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- .changeset/sweet-olives-return.md | 5 ++ .../implementations/rootLoggerService.ts | 6 +- packages/backend-plugin-api/api-report.md | 72 ++++++++++++------- .../services/definitions/cacheServiceRef.ts | 5 +- .../services/definitions/configServiceRef.ts | 7 +- .../definitions/databaseServiceRef.ts | 5 +- .../definitions/discoveryServiceRef.ts | 5 +- .../src/services/definitions/index.ts | 17 +++-- .../definitions/lifecycleServiceRef.ts | 8 +-- .../services/definitions/loggerServiceRef.ts | 6 +- .../definitions/permissionsServiceRef.ts | 7 +- .../definitions/pluginMetadataServiceRef.ts | 10 +-- .../definitions/rootLoggerServiceRef.ts | 7 +- .../definitions/schedulerServiceRef.ts | 5 +- .../definitions/tokenManagerServiceRef.ts | 5 +- .../definitions/urlReaderServiceRef.ts | 5 +- .../services/helpers/loggerToWinstonLogger.ts | 6 +- 17 files changed, 124 insertions(+), 57 deletions(-) create mode 100644 .changeset/sweet-olives-return.md diff --git a/.changeset/sweet-olives-return.md b/.changeset/sweet-olives-return.md new file mode 100644 index 0000000000..9ce2aa49d1 --- /dev/null +++ b/.changeset/sweet-olives-return.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-plugin-api': minor +--- + +**BREAKING**: All service interfaces are now suffixed with `*Service`. diff --git a/packages/backend-app-api/src/services/implementations/rootLoggerService.ts b/packages/backend-app-api/src/services/implementations/rootLoggerService.ts index d2bb406adb..064091a122 100644 --- a/packages/backend-app-api/src/services/implementations/rootLoggerService.ts +++ b/packages/backend-app-api/src/services/implementations/rootLoggerService.ts @@ -17,12 +17,12 @@ import { createRootLogger } from '@backstage/backend-common'; import { createServiceFactory, - Logger, + LoggerService, coreServices, } from '@backstage/backend-plugin-api'; import { Logger as WinstonLogger } from 'winston'; -class BackstageLogger implements Logger { +class BackstageLogger implements LoggerService { static fromWinston(logger: WinstonLogger): BackstageLogger { return new BackstageLogger(logger); } @@ -33,7 +33,7 @@ class BackstageLogger implements Logger { this.winston.info(message, ...meta); } - child(fields: { [name: string]: string }): Logger { + child(fields: { [name: string]: string }): LoggerService { return new BackstageLogger(this.winston.child(fields)); } } diff --git a/packages/backend-plugin-api/api-report.md b/packages/backend-plugin-api/api-report.md index 17cae29bff..d3c1292b7e 100644 --- a/packages/backend-plugin-api/api-report.md +++ b/packages/backend-plugin-api/api-report.md @@ -5,7 +5,7 @@ ```ts import { Config } from '@backstage/config'; import { Handler } from 'express'; -import { Logger as Logger_2 } from 'winston'; +import { Logger } from 'winston'; import { PermissionAuthorizer } from '@backstage/plugin-permission-common'; import { PermissionEvaluator } from '@backstage/plugin-permission-common'; import { PluginCacheManager } from '@backstage/backend-common'; @@ -24,16 +24,6 @@ export interface BackendFeature { register(reg: BackendRegistrationPoints): void; } -// @public (undocumented) -export interface BackendLifecycle { - addShutdownHook(options: BackendLifecycleShutdownHook): void; -} - -// @public (undocumented) -export type BackendLifecycleShutdownHook = { - fn: () => void | Promise; -}; - // @public (undocumented) export interface BackendModuleConfig { // (undocumented) @@ -75,9 +65,15 @@ export interface BackendRegistrationPoints { }): void; } +// @public (undocumented) +export type CacheService = PluginCacheManager; + // @public (undocumented) const cacheServiceRef: ServiceRef; +// @public (undocumented) +export type ConfigService = Config; + // @public (undocumented) const configServiceRef: ServiceRef; @@ -164,9 +160,15 @@ export function createServiceRef(options: { ) => Promise | (() => ServiceFactory)>; }): ServiceRef; +// @public (undocumented) +export type DatabaseService = PluginDatabaseManager; + // @public (undocumented) const databaseServiceRef: ServiceRef; +// @public (undocumented) +export type DiscoveryService = PluginEndpointDiscovery; + // @public (undocumented) const discoveryServiceRef: ServiceRef; @@ -188,42 +190,58 @@ export interface HttpRouterService { const httpRouterServiceRef: ServiceRef; // @public (undocumented) -const lifecycleServiceRef: ServiceRef; +export interface LifecycleService { + addShutdownHook(options: LifecycleServiceShutdownHook): void; +} // @public (undocumented) -export interface Logger { +const lifecycleServiceRef: ServiceRef; + +// @public (undocumented) +export type LifecycleServiceShutdownHook = { + fn: () => void | Promise; +}; + +// @public (undocumented) +export interface LoggerService { // (undocumented) - child(fields: { [name: string]: string }): Logger; + child(fields: { [name: string]: string }): LoggerService; // (undocumented) info(message: string): void; } // @public (undocumented) -const loggerServiceRef: ServiceRef; +const loggerServiceRef: ServiceRef; // @public (undocumented) export function loggerToWinstonLogger( - logger: Logger, + logger: LoggerService, opts?: TransportStreamOptions, -): Logger_2; +): Logger; // @public (undocumented) -const permissionsServiceRef: ServiceRef< - PermissionAuthorizer | PermissionEvaluator, - 'plugin' ->; +export type PermissionsService = PermissionEvaluator | PermissionAuthorizer; // @public (undocumented) -export interface PluginMetadata { +const permissionsServiceRef: ServiceRef; + +// @public (undocumented) +export interface PluginMetadataService { // (undocumented) getId(): string; } // @public (undocumented) -const pluginMetadataServiceRef: ServiceRef; +const pluginMetadataServiceRef: ServiceRef; // @public (undocumented) -const rootLoggerServiceRef: ServiceRef; +export type RootLoggerService = LoggerService; + +// @public (undocumented) +const rootLoggerServiceRef: ServiceRef; + +// @public (undocumented) +export type SchedulerService = PluginTaskScheduler; // @public (undocumented) const schedulerServiceRef: ServiceRef; @@ -267,6 +285,9 @@ export type ServiceRef< $$ref: 'service'; }; +// @public (undocumented) +export type TokenManagerService = TokenManager; + // @public (undocumented) const tokenManagerServiceRef: ServiceRef; @@ -275,6 +296,9 @@ export type TypesToServiceRef = { [key in keyof T]: ServiceRef; }; +// @public (undocumented) +export type UrlReaderService = UrlReader; + // @public (undocumented) const urlReaderServiceRef: ServiceRef; ``` diff --git a/packages/backend-plugin-api/src/services/definitions/cacheServiceRef.ts b/packages/backend-plugin-api/src/services/definitions/cacheServiceRef.ts index 0b572f1712..aff913b06a 100644 --- a/packages/backend-plugin-api/src/services/definitions/cacheServiceRef.ts +++ b/packages/backend-plugin-api/src/services/definitions/cacheServiceRef.ts @@ -17,9 +17,12 @@ import { createServiceRef } from '../system/types'; import { PluginCacheManager } from '@backstage/backend-common'; +/** @public */ +export type CacheService = PluginCacheManager; + /** * @public */ -export const cacheServiceRef = createServiceRef({ +export const cacheServiceRef = createServiceRef({ id: 'core.cache', }); diff --git a/packages/backend-plugin-api/src/services/definitions/configServiceRef.ts b/packages/backend-plugin-api/src/services/definitions/configServiceRef.ts index f17c5f57bc..15e9cb9aa6 100644 --- a/packages/backend-plugin-api/src/services/definitions/configServiceRef.ts +++ b/packages/backend-plugin-api/src/services/definitions/configServiceRef.ts @@ -20,7 +20,12 @@ import { createServiceRef } from '../system/types'; /** * @public */ -export const configServiceRef = createServiceRef({ +export type ConfigService = Config; + +/** + * @public + */ +export const configServiceRef = createServiceRef({ id: 'core.root.config', scope: 'root', }); diff --git a/packages/backend-plugin-api/src/services/definitions/databaseServiceRef.ts b/packages/backend-plugin-api/src/services/definitions/databaseServiceRef.ts index b41159dc9e..aaea5d304f 100644 --- a/packages/backend-plugin-api/src/services/definitions/databaseServiceRef.ts +++ b/packages/backend-plugin-api/src/services/definitions/databaseServiceRef.ts @@ -17,9 +17,12 @@ import { PluginDatabaseManager } from '@backstage/backend-common'; import { createServiceRef } from '../system/types'; +/** @public */ +export type DatabaseService = PluginDatabaseManager; + /** * @public */ -export const databaseServiceRef = createServiceRef({ +export const databaseServiceRef = createServiceRef({ id: 'core.database', }); diff --git a/packages/backend-plugin-api/src/services/definitions/discoveryServiceRef.ts b/packages/backend-plugin-api/src/services/definitions/discoveryServiceRef.ts index 675ac206a2..41505e7756 100644 --- a/packages/backend-plugin-api/src/services/definitions/discoveryServiceRef.ts +++ b/packages/backend-plugin-api/src/services/definitions/discoveryServiceRef.ts @@ -17,9 +17,12 @@ import { createServiceRef } from '../system/types'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; +/** @public */ +export type DiscoveryService = PluginEndpointDiscovery; + /** * @public */ -export const discoveryServiceRef = createServiceRef({ +export const discoveryServiceRef = createServiceRef({ id: 'core.discovery', }); diff --git a/packages/backend-plugin-api/src/services/definitions/index.ts b/packages/backend-plugin-api/src/services/definitions/index.ts index 1ff33362fa..e2f361ff01 100644 --- a/packages/backend-plugin-api/src/services/definitions/index.ts +++ b/packages/backend-plugin-api/src/services/definitions/index.ts @@ -17,10 +17,19 @@ import * as coreServices from './coreServices'; export { coreServices }; +export type { CacheService } from './cacheServiceRef'; +export type { ConfigService } from './configServiceRef'; +export type { DatabaseService } from './databaseServiceRef'; +export type { DiscoveryService } from './discoveryServiceRef'; export type { HttpRouterService } from './httpRouterServiceRef'; -export type { Logger } from './loggerServiceRef'; export type { - BackendLifecycle, - BackendLifecycleShutdownHook, + LifecycleService, + LifecycleServiceShutdownHook, } from './lifecycleServiceRef'; -export type { PluginMetadata } from './pluginMetadataServiceRef'; +export type { LoggerService } from './loggerServiceRef'; +export type { PermissionsService } from './permissionsServiceRef'; +export type { PluginMetadataService } from './pluginMetadataServiceRef'; +export type { RootLoggerService } from './rootLoggerServiceRef'; +export type { SchedulerService } from './schedulerServiceRef'; +export type { TokenManagerService } from './tokenManagerServiceRef'; +export type { UrlReaderService } from './urlReaderServiceRef'; diff --git a/packages/backend-plugin-api/src/services/definitions/lifecycleServiceRef.ts b/packages/backend-plugin-api/src/services/definitions/lifecycleServiceRef.ts index 15610e2643..4061cc1da2 100644 --- a/packages/backend-plugin-api/src/services/definitions/lifecycleServiceRef.ts +++ b/packages/backend-plugin-api/src/services/definitions/lifecycleServiceRef.ts @@ -19,24 +19,24 @@ import { createServiceRef } from '../system/types'; /** * @public **/ -export type BackendLifecycleShutdownHook = { +export type LifecycleServiceShutdownHook = { fn: () => void | Promise; }; /** * @public **/ -export interface BackendLifecycle { +export interface LifecycleService { /** * Register a function to be called when the backend is shutting down. */ - addShutdownHook(options: BackendLifecycleShutdownHook): void; + addShutdownHook(options: LifecycleServiceShutdownHook): void; } /** * @public */ -export const lifecycleServiceRef = createServiceRef({ +export const lifecycleServiceRef = createServiceRef({ id: 'core.lifecycle', scope: 'plugin', }); diff --git a/packages/backend-plugin-api/src/services/definitions/loggerServiceRef.ts b/packages/backend-plugin-api/src/services/definitions/loggerServiceRef.ts index e99bc4e7d4..b135ccc8e6 100644 --- a/packages/backend-plugin-api/src/services/definitions/loggerServiceRef.ts +++ b/packages/backend-plugin-api/src/services/definitions/loggerServiceRef.ts @@ -19,14 +19,14 @@ import { createServiceRef } from '../system/types'; /** * @public */ -export interface Logger { +export interface LoggerService { info(message: string): void; - child(fields: { [name: string]: string }): Logger; + child(fields: { [name: string]: string }): LoggerService; } /** * @public */ -export const loggerServiceRef = createServiceRef({ +export const loggerServiceRef = createServiceRef({ id: 'core.logger', }); diff --git a/packages/backend-plugin-api/src/services/definitions/permissionsServiceRef.ts b/packages/backend-plugin-api/src/services/definitions/permissionsServiceRef.ts index b13ca9e240..b24b2f5f46 100644 --- a/packages/backend-plugin-api/src/services/definitions/permissionsServiceRef.ts +++ b/packages/backend-plugin-api/src/services/definitions/permissionsServiceRef.ts @@ -20,11 +20,12 @@ import { PermissionEvaluator, } from '@backstage/plugin-permission-common'; +/** @public */ +export type PermissionsService = PermissionEvaluator | PermissionAuthorizer; + /** * @public */ -export const permissionsServiceRef = createServiceRef< - PermissionEvaluator | PermissionAuthorizer ->({ +export const permissionsServiceRef = createServiceRef({ id: 'core.permissions', }); diff --git a/packages/backend-plugin-api/src/services/definitions/pluginMetadataServiceRef.ts b/packages/backend-plugin-api/src/services/definitions/pluginMetadataServiceRef.ts index 3af6e54900..4c1ecb130e 100644 --- a/packages/backend-plugin-api/src/services/definitions/pluginMetadataServiceRef.ts +++ b/packages/backend-plugin-api/src/services/definitions/pluginMetadataServiceRef.ts @@ -19,13 +19,15 @@ import { createServiceRef } from '../system/types'; /** * @public */ -export interface PluginMetadata { +export interface PluginMetadataService { getId(): string; } /** * @public */ -export const pluginMetadataServiceRef = createServiceRef({ - id: 'core.plugin-metadata', -}); +export const pluginMetadataServiceRef = createServiceRef( + { + id: 'core.plugin-metadata', + }, +); diff --git a/packages/backend-plugin-api/src/services/definitions/rootLoggerServiceRef.ts b/packages/backend-plugin-api/src/services/definitions/rootLoggerServiceRef.ts index 62e22c53d9..af26ce8899 100644 --- a/packages/backend-plugin-api/src/services/definitions/rootLoggerServiceRef.ts +++ b/packages/backend-plugin-api/src/services/definitions/rootLoggerServiceRef.ts @@ -15,12 +15,15 @@ */ import { createServiceRef } from '../system/types'; -import { Logger } from './loggerServiceRef'; +import { LoggerService } from './loggerServiceRef'; + +/** @public */ +export type RootLoggerService = LoggerService; /** * @public */ -export const rootLoggerServiceRef = createServiceRef({ +export const rootLoggerServiceRef = createServiceRef({ id: 'core.root.logger', scope: 'root', }); diff --git a/packages/backend-plugin-api/src/services/definitions/schedulerServiceRef.ts b/packages/backend-plugin-api/src/services/definitions/schedulerServiceRef.ts index ca442dd34d..160f2fbb93 100644 --- a/packages/backend-plugin-api/src/services/definitions/schedulerServiceRef.ts +++ b/packages/backend-plugin-api/src/services/definitions/schedulerServiceRef.ts @@ -17,9 +17,12 @@ import { createServiceRef } from '../system/types'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; +/** @public */ +export type SchedulerService = PluginTaskScheduler; + /** * @public */ -export const schedulerServiceRef = createServiceRef({ +export const schedulerServiceRef = createServiceRef({ id: 'core.scheduler', }); diff --git a/packages/backend-plugin-api/src/services/definitions/tokenManagerServiceRef.ts b/packages/backend-plugin-api/src/services/definitions/tokenManagerServiceRef.ts index 6995033e81..4b12acdc58 100644 --- a/packages/backend-plugin-api/src/services/definitions/tokenManagerServiceRef.ts +++ b/packages/backend-plugin-api/src/services/definitions/tokenManagerServiceRef.ts @@ -17,9 +17,12 @@ import { createServiceRef } from '../system/types'; import { TokenManager } from '@backstage/backend-common'; +/** @public */ +export type TokenManagerService = TokenManager; + /** * @public */ -export const tokenManagerServiceRef = createServiceRef({ +export const tokenManagerServiceRef = createServiceRef({ id: 'core.tokenManager', }); diff --git a/packages/backend-plugin-api/src/services/definitions/urlReaderServiceRef.ts b/packages/backend-plugin-api/src/services/definitions/urlReaderServiceRef.ts index ebb32f1aed..4fc3680763 100644 --- a/packages/backend-plugin-api/src/services/definitions/urlReaderServiceRef.ts +++ b/packages/backend-plugin-api/src/services/definitions/urlReaderServiceRef.ts @@ -17,9 +17,12 @@ import { createServiceRef } from '../system/types'; import { UrlReader } from '@backstage/backend-common'; +/** @public */ +export type UrlReaderService = UrlReader; + /** * @public */ -export const urlReaderServiceRef = createServiceRef({ +export const urlReaderServiceRef = createServiceRef({ id: 'core.urlReader', }); diff --git a/packages/backend-plugin-api/src/services/helpers/loggerToWinstonLogger.ts b/packages/backend-plugin-api/src/services/helpers/loggerToWinstonLogger.ts index b9ccd70d3f..418f669a75 100644 --- a/packages/backend-plugin-api/src/services/helpers/loggerToWinstonLogger.ts +++ b/packages/backend-plugin-api/src/services/helpers/loggerToWinstonLogger.ts @@ -14,13 +14,13 @@ * limitations under the License. */ -import { Logger as BackstageLogger } from '../definitions'; +import { LoggerService } from '../definitions'; import { Logger as WinstonLogger, createLogger } from 'winston'; import Transport, { TransportStreamOptions } from 'winston-transport'; class BackstageLoggerTransport extends Transport { constructor( - private readonly backstageLogger: BackstageLogger, + private readonly backstageLogger: LoggerService, opts?: TransportStreamOptions, ) { super(opts); @@ -35,7 +35,7 @@ class BackstageLoggerTransport extends Transport { /** @public */ export function loggerToWinstonLogger( - logger: BackstageLogger, + logger: LoggerService, opts?: TransportStreamOptions, ): WinstonLogger { return createLogger({ From 05a928e296ca0d855bc7710288412cf29eaa8478 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 14 Dec 2022 14:39:39 +0100 Subject: [PATCH 090/237] fix breakages from Service interface renames Signed-off-by: Patrik Oldsberg --- .changeset/seven-tomatoes-deny.md | 6 ++++++ .../services/implementations/lifecycleService.ts | 12 ++++++------ .../src/module/WrapperProviders.ts | 15 +++++++++------ 3 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 .changeset/seven-tomatoes-deny.md diff --git a/.changeset/seven-tomatoes-deny.md b/.changeset/seven-tomatoes-deny.md new file mode 100644 index 0000000000..f94c97f847 --- /dev/null +++ b/.changeset/seven-tomatoes-deny.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-app-api': patch +'@backstage/plugin-catalog-backend-module-incremental-ingestion': patch +--- + +Updated usages of types from `@backstage/backend-plugin-api`. diff --git a/packages/backend-app-api/src/services/implementations/lifecycleService.ts b/packages/backend-app-api/src/services/implementations/lifecycleService.ts index 5616296e93..317a2e45b8 100644 --- a/packages/backend-app-api/src/services/implementations/lifecycleService.ts +++ b/packages/backend-app-api/src/services/implementations/lifecycleService.ts @@ -14,11 +14,11 @@ * limitations under the License. */ import { - BackendLifecycle, + LifecycleService, createServiceFactory, coreServices, loggerToWinstonLogger, - BackendLifecycleShutdownHook, + LifecycleServiceShutdownHook, } from '@backstage/backend-plugin-api'; import { Logger } from 'winston'; @@ -29,11 +29,11 @@ export class BackendLifecycleImpl { } #isCalled = false; - #shutdownTasks: Array = + #shutdownTasks: Array = []; addShutdownHook( - options: BackendLifecycleShutdownHook & { pluginId: string }, + options: LifecycleServiceShutdownHook & { pluginId: string }, ): void { this.#shutdownTasks.push(options); } @@ -64,12 +64,12 @@ export class BackendLifecycleImpl { } } -class PluginScopedLifecycleImpl implements BackendLifecycle { +class PluginScopedLifecycleImpl implements LifecycleService { constructor( private readonly lifecycle: BackendLifecycleImpl, private readonly pluginId: string, ) {} - addShutdownHook(options: BackendLifecycleShutdownHook): void { + addShutdownHook(options: LifecycleServiceShutdownHook): void { this.lifecycle.addShutdownHook({ ...options, pluginId: this.pluginId }); } } diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts index 44da5067fd..a6cfe14df2 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts @@ -14,9 +14,12 @@ * limitations under the License. */ -import { Logger, loggerToWinstonLogger } from '@backstage/backend-plugin-api'; -import { PluginTaskScheduler } from '@backstage/backend-tasks'; -import { Config } from '@backstage/config'; +import { + ConfigService, + LoggerService, + SchedulerService, + loggerToWinstonLogger, +} from '@backstage/backend-plugin-api'; import { stringifyError } from '@backstage/errors'; import { EntityProvider, @@ -46,10 +49,10 @@ export class WrapperProviders { constructor( private readonly options: { - config: Config; - logger: Logger; + config: ConfigService; + logger: LoggerService; client: Knex; - scheduler: PluginTaskScheduler; + scheduler: SchedulerService; applyDatabaseMigrations?: typeof applyDatabaseMigrations; }, ) {} From fe1d2269cde1c513174cd918fd94c1d665890ed3 Mon Sep 17 00:00:00 2001 From: djamaile Date: Wed, 14 Dec 2022 15:06:07 +0100 Subject: [PATCH 091/237] chore: use require resolve Signed-off-by: djamaile --- .../repo-tools/src/commands/api-reports/api-extractor.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index 43ef750322..6441de2770 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -66,8 +66,6 @@ import { import { IMarkdownEmitterContext } from '@microsoft/api-documenter/lib/markdown/MarkdownEmitter'; import { AstDeclaration } from '@microsoft/api-extractor/lib/analyzer/AstDeclaration'; import { paths as cliPaths } from '../../lib/paths'; -import { resolvePackagePath as resolvePackagePathBackend } from '@backstage/backend-common'; - import minimatch from 'minimatch'; const tmpDir = cliPaths.resolveTargetRoot( @@ -233,10 +231,7 @@ export async function createTemporaryTsConfig(includedPackageDirs: string[]) { extends: './tsconfig.json', include: [ // These two contain global definitions that are needed for stable API report generation - resolvePackagePathBackend( - '@backstage/cli', - 'asset-types/asset-types.d.ts', - ), + require.resolve('@backstage/cli/asset-types/asset-types.d.ts'), ...includedPackageDirs.map(dir => join(dir, 'src')), ], // we don't exclude node_modules so that we can use the asset-types.d.ts file From e3dfef3f6396b0b8804c5aaa8b7aa681217bc15f Mon Sep 17 00:00:00 2001 From: Gabrielle Langer Date: Wed, 14 Dec 2022 15:51:31 +0100 Subject: [PATCH 092/237] fix(plugin-lighthouse): fix form factor field in the audit creation form not working with the latest version of lighthouse-audit-service Signed-off-by: Gabrielle Langer --- .changeset/many-mayflies-share.md | 5 +++ plugins/lighthouse/api-report.md | 21 ++++++++++- plugins/lighthouse/src/api.ts | 21 ++++++++++- .../src/components/CreateAudit/index.tsx | 37 ++++++++++++------- 4 files changed, 69 insertions(+), 15 deletions(-) create mode 100644 .changeset/many-mayflies-share.md diff --git a/.changeset/many-mayflies-share.md b/.changeset/many-mayflies-share.md new file mode 100644 index 0000000000..080afc6ac1 --- /dev/null +++ b/.changeset/many-mayflies-share.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-lighthouse': patch +--- + +Fixed "Emulated Form Factor" field in the audit creation form not working with the latest version (1.0.2) of `lighthouse-audit-service`. diff --git a/plugins/lighthouse/api-report.md b/plugins/lighthouse/api-report.md index 17c771e187..e4d945b6dd 100644 --- a/plugins/lighthouse/api-report.md +++ b/plugins/lighthouse/api-report.md @@ -71,6 +71,14 @@ export class FetchError extends Error { get name(): string; } +// @public (undocumented) +export enum FormFactor { + // (undocumented) + Desktop = 'desktop', + // (undocumented) + Mobile = 'mobile', +} + // @public (undocumented) const isLighthouseAvailable: (entity: Entity) => boolean; export { isLighthouseAvailable }; @@ -167,13 +175,24 @@ export class LighthouseRestApi implements LighthouseApi { // @public (undocumented) export const Router: () => JSX.Element; +// @public (undocumented) +export type ScreenEmulation = { + mobile: boolean; + width: number; + height: number; + deviceScaleFactor: number; + disabled: boolean; +} | null; + // @public (undocumented) export interface TriggerAuditPayload { // (undocumented) options: { lighthouseConfig: { settings: { - emulatedFormFactor: string; + formFactor: FormFactor; + screenEmulation: ScreenEmulation; + emulatedFormFactor: FormFactor; }; }; }; diff --git a/plugins/lighthouse/src/api.ts b/plugins/lighthouse/src/api.ts index e4ccefe2a1..0bad997087 100644 --- a/plugins/lighthouse/src/api.ts +++ b/plugins/lighthouse/src/api.ts @@ -85,13 +85,32 @@ export interface Website { /** @public */ export type WebsiteListResponse = LASListResponse; +/** @public */ +export enum FormFactor { + Mobile = 'mobile', + Desktop = 'desktop', +} + +/** @public */ +export type ScreenEmulation = { + mobile: boolean; + width: number; + height: number; + deviceScaleFactor: number; + disabled: boolean; +} | null; + /** @public */ export interface TriggerAuditPayload { url: string; options: { lighthouseConfig: { settings: { - emulatedFormFactor: string; + // For lighthouse 7+ + formFactor: FormFactor; + screenEmulation: ScreenEmulation; + // For lighthouse before 7 + emulatedFormFactor: FormFactor; }; }; }; diff --git a/plugins/lighthouse/src/components/CreateAudit/index.tsx b/plugins/lighthouse/src/components/CreateAudit/index.tsx index f83ad6a8a3..2ee5a675b6 100644 --- a/plugins/lighthouse/src/components/CreateAudit/index.tsx +++ b/plugins/lighthouse/src/components/CreateAudit/index.tsx @@ -25,7 +25,7 @@ import { } from '@material-ui/core'; import React, { useCallback, useState } from 'react'; import { useNavigate } from 'react-router-dom'; -import { lighthouseApiRef } from '../../api'; +import { FormFactor, lighthouseApiRef, ScreenEmulation } from '../../api'; import { useQuery } from '../../utils'; import LighthouseSupportButton from '../SupportButton'; @@ -65,6 +65,20 @@ const useStyles = makeStyles(theme => ({ }, })); +const formFactorToScreenEmulationMap: Record = { + // the default is mobile, so no need to override + mobile: null, + // Values from lighthouse's cli "desktop" preset + // https://github.com/GoogleChrome/lighthouse/blob/a6738e0033e7e5ca308b97c1c36f298b7d399402/lighthouse-core/config/constants.js#L71-L77 + desktop: { + mobile: false, + width: 1350, + height: 940, + deviceScaleFactor: 1, + disabled: false, + }, +}; + export const CreateAuditContent = () => { const errorApi = useApi(errorApiRef); const lighthouseApi = useApi(lighthouseApiRef); @@ -73,7 +87,7 @@ export const CreateAuditContent = () => { const navigate = useNavigate(); const [submitting, setSubmitting] = useState(false); const [url, setUrl] = useState(query.get('url') || ''); - const [emulatedFormFactor, setEmulatedFormFactor] = useState('mobile'); + const [formFactor, setFormFactor] = useState(FormFactor.Mobile); const triggerAudit = useCallback(async (): Promise => { setSubmitting(true); @@ -85,7 +99,9 @@ export const CreateAuditContent = () => { options: { lighthouseConfig: { settings: { - emulatedFormFactor, + formFactor, + emulatedFormFactor: formFactor, + screenEmulation: formFactorToScreenEmulationMap[formFactor], }, }, }, @@ -96,14 +112,7 @@ export const CreateAuditContent = () => { } finally { setSubmitting(false); } - }, [ - url, - emulatedFormFactor, - lighthouseApi, - setSubmitting, - errorApi, - navigate, - ]); + }, [url, formFactor, lighthouseApi, setSubmitting, errorApi, navigate]); return ( <> @@ -146,8 +155,10 @@ export const CreateAuditContent = () => { select required disabled={submitting} - onChange={ev => setEmulatedFormFactor(ev.target.value)} - value={emulatedFormFactor} + onChange={ev => + setFormFactor(ev.target.value as FormFactor) + } + value={formFactor} inputProps={{ 'aria-label': 'Emulated form factor' }} > Mobile From 824046de09f334876fd74a238fb401a0abe3a4ef Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 14 Dec 2022 16:13:24 +0000 Subject: [PATCH 093/237] Update dependency @roadiehq/backstage-plugin-github-insights to v2.2.2 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 1cda4df54c..9ee779f2a5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12779,8 +12779,8 @@ __metadata: linkType: hard "@roadiehq/backstage-plugin-github-insights@npm:^2.0.5": - version: 2.2.1 - resolution: "@roadiehq/backstage-plugin-github-insights@npm:2.2.1" + version: 2.2.2 + resolution: "@roadiehq/backstage-plugin-github-insights@npm:2.2.2" dependencies: "@backstage/catalog-model": ^1.1.3 "@backstage/core-components": ^0.12.0 @@ -12802,7 +12802,7 @@ __metadata: react: ^16.13.1 || ^17.0.0 react-dom: ^16.13.1 || ^17.0.0 react-router: 6.0.0-beta.0 || ^6.3.0 - checksum: 4ef3c8d7da22a2cd1c5015e9e4d0782c60e5842f92fb072cc2b61f441fcfd10aa5ca6bda3e489600ac425ae9f9b730f88fa171f91ede00fdcd556a916c6d0d3a + checksum: 866d0ec0cb169855c6206e075bea16122450c9be533f4937e3c7f9869d9dc32cf9a770b42b1468078c37484cc3745dc785dcf39431c5a66d6f54573994f20a34 languageName: node linkType: hard From 1fe43f4017cf031d8336320ecb1d752d8db64acc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 14 Dec 2022 17:31:39 +0100 Subject: [PATCH 094/237] backend-app-api: update API report Signed-off-by: Patrik Oldsberg --- packages/backend-app-api/api-report.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/backend-app-api/api-report.md b/packages/backend-app-api/api-report.md index 5008b1efe1..eba4517a1f 100644 --- a/packages/backend-app-api/api-report.md +++ b/packages/backend-app-api/api-report.md @@ -4,13 +4,12 @@ ```ts import { BackendFeature } from '@backstage/backend-plugin-api'; -import { BackendLifecycle } from '@backstage/backend-plugin-api'; import { Config } from '@backstage/config'; import { ExtensionPoint } from '@backstage/backend-plugin-api'; import { HttpRouterService } from '@backstage/backend-plugin-api'; -import { Logger } from '@backstage/backend-plugin-api'; -import { PermissionAuthorizer } from '@backstage/plugin-permission-common'; -import { PermissionEvaluator } from '@backstage/plugin-permission-common'; +import { LifecycleService } from '@backstage/backend-plugin-api'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { PermissionsService } from '@backstage/backend-plugin-api'; import { PluginCacheManager } from '@backstage/backend-common'; import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; @@ -70,18 +69,22 @@ export type HttpRouterFactoryOptions = { // @public export const lifecycleFactory: ( options?: undefined, -) => ServiceFactory; +) => ServiceFactory; // @public (undocumented) -export const loggerFactory: (options?: undefined) => ServiceFactory; +export const loggerFactory: ( + options?: undefined, +) => ServiceFactory; // @public (undocumented) export const permissionsFactory: ( options?: undefined, -) => ServiceFactory; +) => ServiceFactory; // @public (undocumented) -export const rootLoggerFactory: (options?: undefined) => ServiceFactory; +export const rootLoggerFactory: ( + options?: undefined, +) => ServiceFactory; // @public (undocumented) export const schedulerFactory: ( From 3c70fb6dc5b4cd10dc16fe19ad9340aab52f8925 Mon Sep 17 00:00:00 2001 From: Carlos Esteban Lopez Date: Wed, 14 Dec 2022 11:34:13 -0500 Subject: [PATCH 095/237] test: Fix test import error Signed-off-by: Carlos Esteban Lopez --- packages/core-components/src/layout/Sidebar/Items.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core-components/src/layout/Sidebar/Items.test.tsx b/packages/core-components/src/layout/Sidebar/Items.test.tsx index 2a150b301d..eafbf3827e 100644 --- a/packages/core-components/src/layout/Sidebar/Items.test.tsx +++ b/packages/core-components/src/layout/Sidebar/Items.test.tsx @@ -23,7 +23,7 @@ import CreateComponentIcon from '@material-ui/icons/AddCircleOutline'; import { Sidebar } from './Bar'; import { SidebarItem, SidebarSearchField, SidebarExpandButton } from './Items'; import { renderHook } from '@testing-library/react-hooks'; -import { hexToRgb, makeStyles } from '@material-ui/core/styles'; +import { makeStyles } from '@material-ui/core/styles'; const useStyles = makeStyles({ spotlight: { From cb1c2781c053c293c40259e4305a533a343a7393 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 14 Dec 2022 17:20:20 +0100 Subject: [PATCH 096/237] backend-plugin-api: update logger interface with more methods and meta MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- .changeset/chilly-bees-dream.md | 5 +++++ .changeset/wicked-paws-help.md | 5 +++++ .../implementations/rootLoggerService.ts | 21 +++++++++++++++---- .../src/services/definitions/index.ts | 2 +- .../services/definitions/loggerServiceRef.ts | 13 ++++++++++-- 5 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 .changeset/chilly-bees-dream.md create mode 100644 .changeset/wicked-paws-help.md diff --git a/.changeset/chilly-bees-dream.md b/.changeset/chilly-bees-dream.md new file mode 100644 index 0000000000..34e401a199 --- /dev/null +++ b/.changeset/chilly-bees-dream.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-app-api': patch +--- + +Updated logger implementations to match interface changes. diff --git a/.changeset/wicked-paws-help.md b/.changeset/wicked-paws-help.md new file mode 100644 index 0000000000..8fbba71662 --- /dev/null +++ b/.changeset/wicked-paws-help.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-plugin-api': patch +--- + +Updated `LoggerService` interface with more log methods and meta. diff --git a/packages/backend-app-api/src/services/implementations/rootLoggerService.ts b/packages/backend-app-api/src/services/implementations/rootLoggerService.ts index 064091a122..54d40b6a12 100644 --- a/packages/backend-app-api/src/services/implementations/rootLoggerService.ts +++ b/packages/backend-app-api/src/services/implementations/rootLoggerService.ts @@ -20,6 +20,7 @@ import { LoggerService, coreServices, } from '@backstage/backend-plugin-api'; +import { LogMeta } from '@backstage/backend-plugin-api'; import { Logger as WinstonLogger } from 'winston'; class BackstageLogger implements LoggerService { @@ -29,12 +30,24 @@ class BackstageLogger implements LoggerService { private constructor(private readonly winston: WinstonLogger) {} - info(message: string, ...meta: any[]): void { - this.winston.info(message, ...meta); + error(message: string, meta?: LogMeta): void { + this.winston.error(message, meta); } - child(fields: { [name: string]: string }): LoggerService { - return new BackstageLogger(this.winston.child(fields)); + warn(message: string, meta?: LogMeta): void { + this.winston.warn(message, meta); + } + + info(message: string, meta?: LogMeta): void { + this.winston.info(message, meta); + } + + debug(message: string, meta?: LogMeta): void { + this.winston.debug(message, meta); + } + + child(meta: LogMeta): LoggerService { + return new BackstageLogger(this.winston.child(meta)); } } diff --git a/packages/backend-plugin-api/src/services/definitions/index.ts b/packages/backend-plugin-api/src/services/definitions/index.ts index e2f361ff01..3cf2ab4c77 100644 --- a/packages/backend-plugin-api/src/services/definitions/index.ts +++ b/packages/backend-plugin-api/src/services/definitions/index.ts @@ -26,7 +26,7 @@ export type { LifecycleService, LifecycleServiceShutdownHook, } from './lifecycleServiceRef'; -export type { LoggerService } from './loggerServiceRef'; +export type { LoggerService, LogMeta } from './loggerServiceRef'; export type { PermissionsService } from './permissionsServiceRef'; export type { PluginMetadataService } from './pluginMetadataServiceRef'; export type { RootLoggerService } from './rootLoggerServiceRef'; diff --git a/packages/backend-plugin-api/src/services/definitions/loggerServiceRef.ts b/packages/backend-plugin-api/src/services/definitions/loggerServiceRef.ts index b135ccc8e6..179e97a84b 100644 --- a/packages/backend-plugin-api/src/services/definitions/loggerServiceRef.ts +++ b/packages/backend-plugin-api/src/services/definitions/loggerServiceRef.ts @@ -16,12 +16,21 @@ import { createServiceRef } from '../system/types'; +/** + * @public + */ +export type LogMeta = Error | { [name: string]: any }; + /** * @public */ export interface LoggerService { - info(message: string): void; - child(fields: { [name: string]: string }): LoggerService; + error(message: string, meta?: LogMeta): void; + warn(message: string, meta?: LogMeta): void; + info(message: string, meta?: LogMeta): void; + debug(message: string, meta?: LogMeta): void; + + child(meta: LogMeta): LoggerService; } /** From 1d068ea8a4152b9de4e42fc856962dd0fe00d827 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 14 Dec 2022 17:25:25 +0100 Subject: [PATCH 097/237] backend-plugin-api: update loggerToWinstonLogger to support log levels Signed-off-by: Patrik Oldsberg --- .../services/helpers/loggerToWinstonLogger.ts | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/backend-plugin-api/src/services/helpers/loggerToWinstonLogger.ts b/packages/backend-plugin-api/src/services/helpers/loggerToWinstonLogger.ts index 418f669a75..1499b4e7ef 100644 --- a/packages/backend-plugin-api/src/services/helpers/loggerToWinstonLogger.ts +++ b/packages/backend-plugin-api/src/services/helpers/loggerToWinstonLogger.ts @@ -26,9 +26,24 @@ class BackstageLoggerTransport extends Transport { super(opts); } - log(info: { message: string }, callback: VoidFunction) { - // TODO: add support for levels and fields - this.backstageLogger.info(info.message); + log(info: any, callback: VoidFunction) { + const { level, message, ...meta } = info; + switch (level) { + case 'error': + this.backstageLogger.error(message, meta); + break; + case 'warn': + this.backstageLogger.warn(message, meta); + break; + case 'info': + this.backstageLogger.info(message, meta); + break; + case 'debug': + this.backstageLogger.debug(message, meta); + break; + default: + this.backstageLogger.info(message, meta); + } callback(); } } From 2ee44029f9d87afa2bfc22e94253264070128ca2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 14 Dec 2022 17:31:14 +0100 Subject: [PATCH 098/237] backend-plugin-api: update API report Signed-off-by: Patrik Oldsberg --- packages/backend-plugin-api/api-report.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/backend-plugin-api/api-report.md b/packages/backend-plugin-api/api-report.md index d3c1292b7e..bbc0a39518 100644 --- a/packages/backend-plugin-api/api-report.md +++ b/packages/backend-plugin-api/api-report.md @@ -205,9 +205,15 @@ export type LifecycleServiceShutdownHook = { // @public (undocumented) export interface LoggerService { // (undocumented) - child(fields: { [name: string]: string }): LoggerService; + child(meta: LogMeta): LoggerService; // (undocumented) - info(message: string): void; + debug(message: string, meta?: LogMeta): void; + // (undocumented) + error(message: string, meta?: LogMeta): void; + // (undocumented) + info(message: string, meta?: LogMeta): void; + // (undocumented) + warn(message: string, meta?: LogMeta): void; } // @public (undocumented) @@ -219,6 +225,13 @@ export function loggerToWinstonLogger( opts?: TransportStreamOptions, ): Logger; +// @public (undocumented) +export type LogMeta = + | Error + | { + [name: string]: any; + }; + // @public (undocumented) export type PermissionsService = PermissionEvaluator | PermissionAuthorizer; From 2dbe5f00c545258f11da01741c3b3362ed4bc7d8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 14 Dec 2022 18:00:32 +0100 Subject: [PATCH 099/237] changesets: update changeset to reflect startTestBackend updates Signed-off-by: Patrik Oldsberg --- .changeset/green-moose-leave.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/green-moose-leave.md b/.changeset/green-moose-leave.md index 23e4b6878f..df6ff04d82 100644 --- a/.changeset/green-moose-leave.md +++ b/.changeset/green-moose-leave.md @@ -2,4 +2,4 @@ '@backstage/backend-test-utils': patch --- -Backends started with `startTestBackend` are now automatically stopped after the each test, unless the `autoStop` option is set to `false`. +Backends started with `startTestBackend` are now automatically stopped after all tests have run. From 341927c47e84b4e67251e0ae3ede3f14b5af8f07 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 14 Dec 2022 18:00:51 +0100 Subject: [PATCH 100/237] backend-test-utils: stop all running test instances concurrently Signed-off-by: Patrik Oldsberg --- .../src/next/wiring/TestBackend.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.ts index 7dbcf439dc..5558a74ca1 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.ts @@ -138,13 +138,15 @@ function registerTestHooks() { registered = true; afterAll(async () => { - for (const backend of backendInstancesToCleanUp) { - try { - await backend.stop(); - } catch (error) { - console.error(`Failed to stop backend after tests, ${error}`); - } - } + await Promise.all( + backendInstancesToCleanUp.map(async backend => { + try { + await backend.stop(); + } catch (error) { + console.error(`Failed to stop backend after tests, ${error}`); + } + }), + ); backendInstancesToCleanUp.length = 0; }); } From a9d0438c111e48285fbb1393565b5d7f059f1281 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 14 Dec 2022 17:02:18 +0000 Subject: [PATCH 101/237] Update dependency @swc/core to v1.3.23 Signed-off-by: Renovate Bot --- storybook/yarn.lock | 86 ++++++++++++++++++++++----------------------- yarn.lock | 86 ++++++++++++++++++++++----------------------- 2 files changed, 86 insertions(+), 86 deletions(-) diff --git a/storybook/yarn.lock b/storybook/yarn.lock index 033225c60d..ef593ea7c8 100644 --- a/storybook/yarn.lock +++ b/storybook/yarn.lock @@ -2966,90 +2966,90 @@ __metadata: languageName: node linkType: hard -"@swc/core-darwin-arm64@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-darwin-arm64@npm:1.3.22" +"@swc/core-darwin-arm64@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-darwin-arm64@npm:1.3.23" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@swc/core-darwin-x64@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-darwin-x64@npm:1.3.22" +"@swc/core-darwin-x64@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-darwin-x64@npm:1.3.23" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@swc/core-linux-arm-gnueabihf@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-linux-arm-gnueabihf@npm:1.3.22" +"@swc/core-linux-arm-gnueabihf@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-linux-arm-gnueabihf@npm:1.3.23" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@swc/core-linux-arm64-gnu@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-linux-arm64-gnu@npm:1.3.22" +"@swc/core-linux-arm64-gnu@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-linux-arm64-gnu@npm:1.3.23" conditions: os=linux & cpu=arm64 & libc=glibc languageName: node linkType: hard -"@swc/core-linux-arm64-musl@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-linux-arm64-musl@npm:1.3.22" +"@swc/core-linux-arm64-musl@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-linux-arm64-musl@npm:1.3.23" conditions: os=linux & cpu=arm64 & libc=musl languageName: node linkType: hard -"@swc/core-linux-x64-gnu@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-linux-x64-gnu@npm:1.3.22" +"@swc/core-linux-x64-gnu@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-linux-x64-gnu@npm:1.3.23" conditions: os=linux & cpu=x64 & libc=glibc languageName: node linkType: hard -"@swc/core-linux-x64-musl@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-linux-x64-musl@npm:1.3.22" +"@swc/core-linux-x64-musl@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-linux-x64-musl@npm:1.3.23" conditions: os=linux & cpu=x64 & libc=musl languageName: node linkType: hard -"@swc/core-win32-arm64-msvc@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-win32-arm64-msvc@npm:1.3.22" +"@swc/core-win32-arm64-msvc@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-win32-arm64-msvc@npm:1.3.23" conditions: os=win32 & cpu=arm64 languageName: node linkType: hard -"@swc/core-win32-ia32-msvc@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-win32-ia32-msvc@npm:1.3.22" +"@swc/core-win32-ia32-msvc@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-win32-ia32-msvc@npm:1.3.23" conditions: os=win32 & cpu=ia32 languageName: node linkType: hard -"@swc/core-win32-x64-msvc@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-win32-x64-msvc@npm:1.3.22" +"@swc/core-win32-x64-msvc@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-win32-x64-msvc@npm:1.3.23" conditions: os=win32 & cpu=x64 languageName: node linkType: hard "@swc/core@npm:^1.3.9": - version: 1.3.22 - resolution: "@swc/core@npm:1.3.22" + version: 1.3.23 + resolution: "@swc/core@npm:1.3.23" dependencies: - "@swc/core-darwin-arm64": 1.3.22 - "@swc/core-darwin-x64": 1.3.22 - "@swc/core-linux-arm-gnueabihf": 1.3.22 - "@swc/core-linux-arm64-gnu": 1.3.22 - "@swc/core-linux-arm64-musl": 1.3.22 - "@swc/core-linux-x64-gnu": 1.3.22 - "@swc/core-linux-x64-musl": 1.3.22 - "@swc/core-win32-arm64-msvc": 1.3.22 - "@swc/core-win32-ia32-msvc": 1.3.22 - "@swc/core-win32-x64-msvc": 1.3.22 + "@swc/core-darwin-arm64": 1.3.23 + "@swc/core-darwin-x64": 1.3.23 + "@swc/core-linux-arm-gnueabihf": 1.3.23 + "@swc/core-linux-arm64-gnu": 1.3.23 + "@swc/core-linux-arm64-musl": 1.3.23 + "@swc/core-linux-x64-gnu": 1.3.23 + "@swc/core-linux-x64-musl": 1.3.23 + "@swc/core-win32-arm64-msvc": 1.3.23 + "@swc/core-win32-ia32-msvc": 1.3.23 + "@swc/core-win32-x64-msvc": 1.3.23 dependenciesMeta: "@swc/core-darwin-arm64": optional: true @@ -3073,7 +3073,7 @@ __metadata: optional: true bin: swcx: run_swcx.js - checksum: 5c6fa613502cbfae9985c7e97452649120e50e65695c8243111b422f6dddfebabdbc5ec94a9238e199e6a823a86906635e18e731f8ed45470bca8e4964d11659 + checksum: 2a92949c33f292a369f4a750a8a9759f88f6f399bc0beef54768fce3d1af5cf0c33a4c8f3c2478020ae95e99d99b7c17f301b79163cdbcdf533b91675f1d2e8d languageName: node linkType: hard diff --git a/yarn.lock b/yarn.lock index 9ee779f2a5..2048c5ff5e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13359,90 +13359,90 @@ __metadata: languageName: node linkType: hard -"@swc/core-darwin-arm64@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-darwin-arm64@npm:1.3.22" +"@swc/core-darwin-arm64@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-darwin-arm64@npm:1.3.23" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@swc/core-darwin-x64@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-darwin-x64@npm:1.3.22" +"@swc/core-darwin-x64@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-darwin-x64@npm:1.3.23" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@swc/core-linux-arm-gnueabihf@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-linux-arm-gnueabihf@npm:1.3.22" +"@swc/core-linux-arm-gnueabihf@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-linux-arm-gnueabihf@npm:1.3.23" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@swc/core-linux-arm64-gnu@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-linux-arm64-gnu@npm:1.3.22" +"@swc/core-linux-arm64-gnu@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-linux-arm64-gnu@npm:1.3.23" conditions: os=linux & cpu=arm64 & libc=glibc languageName: node linkType: hard -"@swc/core-linux-arm64-musl@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-linux-arm64-musl@npm:1.3.22" +"@swc/core-linux-arm64-musl@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-linux-arm64-musl@npm:1.3.23" conditions: os=linux & cpu=arm64 & libc=musl languageName: node linkType: hard -"@swc/core-linux-x64-gnu@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-linux-x64-gnu@npm:1.3.22" +"@swc/core-linux-x64-gnu@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-linux-x64-gnu@npm:1.3.23" conditions: os=linux & cpu=x64 & libc=glibc languageName: node linkType: hard -"@swc/core-linux-x64-musl@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-linux-x64-musl@npm:1.3.22" +"@swc/core-linux-x64-musl@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-linux-x64-musl@npm:1.3.23" conditions: os=linux & cpu=x64 & libc=musl languageName: node linkType: hard -"@swc/core-win32-arm64-msvc@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-win32-arm64-msvc@npm:1.3.22" +"@swc/core-win32-arm64-msvc@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-win32-arm64-msvc@npm:1.3.23" conditions: os=win32 & cpu=arm64 languageName: node linkType: hard -"@swc/core-win32-ia32-msvc@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-win32-ia32-msvc@npm:1.3.22" +"@swc/core-win32-ia32-msvc@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-win32-ia32-msvc@npm:1.3.23" conditions: os=win32 & cpu=ia32 languageName: node linkType: hard -"@swc/core-win32-x64-msvc@npm:1.3.22": - version: 1.3.22 - resolution: "@swc/core-win32-x64-msvc@npm:1.3.22" +"@swc/core-win32-x64-msvc@npm:1.3.23": + version: 1.3.23 + resolution: "@swc/core-win32-x64-msvc@npm:1.3.23" conditions: os=win32 & cpu=x64 languageName: node linkType: hard "@swc/core@npm:^1.3.9": - version: 1.3.22 - resolution: "@swc/core@npm:1.3.22" + version: 1.3.23 + resolution: "@swc/core@npm:1.3.23" dependencies: - "@swc/core-darwin-arm64": 1.3.22 - "@swc/core-darwin-x64": 1.3.22 - "@swc/core-linux-arm-gnueabihf": 1.3.22 - "@swc/core-linux-arm64-gnu": 1.3.22 - "@swc/core-linux-arm64-musl": 1.3.22 - "@swc/core-linux-x64-gnu": 1.3.22 - "@swc/core-linux-x64-musl": 1.3.22 - "@swc/core-win32-arm64-msvc": 1.3.22 - "@swc/core-win32-ia32-msvc": 1.3.22 - "@swc/core-win32-x64-msvc": 1.3.22 + "@swc/core-darwin-arm64": 1.3.23 + "@swc/core-darwin-x64": 1.3.23 + "@swc/core-linux-arm-gnueabihf": 1.3.23 + "@swc/core-linux-arm64-gnu": 1.3.23 + "@swc/core-linux-arm64-musl": 1.3.23 + "@swc/core-linux-x64-gnu": 1.3.23 + "@swc/core-linux-x64-musl": 1.3.23 + "@swc/core-win32-arm64-msvc": 1.3.23 + "@swc/core-win32-ia32-msvc": 1.3.23 + "@swc/core-win32-x64-msvc": 1.3.23 dependenciesMeta: "@swc/core-darwin-arm64": optional: true @@ -13466,7 +13466,7 @@ __metadata: optional: true bin: swcx: run_swcx.js - checksum: 5c6fa613502cbfae9985c7e97452649120e50e65695c8243111b422f6dddfebabdbc5ec94a9238e199e6a823a86906635e18e731f8ed45470bca8e4964d11659 + checksum: 2a92949c33f292a369f4a750a8a9759f88f6f399bc0beef54768fce3d1af5cf0c33a4c8f3c2478020ae95e99d99b7c17f301b79163cdbcdf533b91675f1d2e8d languageName: node linkType: hard From 226099ad031a91deb438ea09b99f53ce9fdd3025 Mon Sep 17 00:00:00 2001 From: Gabrielle Langer Date: Wed, 14 Dec 2022 18:10:04 +0100 Subject: [PATCH 102/237] fix(plugin-lighthouse): incorporate feedback and add unit tests Signed-off-by: Gabrielle Langer --- plugins/lighthouse/api-report.md | 37 ++++----- plugins/lighthouse/src/api.ts | 35 ++++---- .../src/components/CreateAudit/index.test.tsx | 82 +++++++++++++++++++ .../src/components/CreateAudit/index.tsx | 15 +++- 4 files changed, 127 insertions(+), 42 deletions(-) diff --git a/plugins/lighthouse/api-report.md b/plugins/lighthouse/api-report.md index e4d945b6dd..e9089c7e92 100644 --- a/plugins/lighthouse/api-report.md +++ b/plugins/lighthouse/api-report.md @@ -72,12 +72,7 @@ export class FetchError extends Error { } // @public (undocumented) -export enum FormFactor { - // (undocumented) - Desktop = 'desktop', - // (undocumented) - Mobile = 'mobile', -} +export type FormFactor = 'mobile' | 'desktop'; // @public (undocumented) const isLighthouseAvailable: (entity: Entity) => boolean; @@ -140,6 +135,21 @@ export type LighthouseCategoryId = | 'accessibility' | 'best-practices'; +// @public (undocumented) +export type LighthouseConfigSettings = { + formFactor: FormFactor; + screenEmulation: + | { + mobile: boolean; + width: number; + height: number; + deviceScaleFactor: number; + disabled: boolean; + } + | undefined; + emulatedFormFactor: FormFactor; +}; + // @public (undocumented) export const LighthousePage: () => JSX.Element; @@ -175,25 +185,12 @@ export class LighthouseRestApi implements LighthouseApi { // @public (undocumented) export const Router: () => JSX.Element; -// @public (undocumented) -export type ScreenEmulation = { - mobile: boolean; - width: number; - height: number; - deviceScaleFactor: number; - disabled: boolean; -} | null; - // @public (undocumented) export interface TriggerAuditPayload { // (undocumented) options: { lighthouseConfig: { - settings: { - formFactor: FormFactor; - screenEmulation: ScreenEmulation; - emulatedFormFactor: FormFactor; - }; + settings: LighthouseConfigSettings; }; }; // (undocumented) diff --git a/plugins/lighthouse/src/api.ts b/plugins/lighthouse/src/api.ts index 0bad997087..a7197c4b96 100644 --- a/plugins/lighthouse/src/api.ts +++ b/plugins/lighthouse/src/api.ts @@ -86,32 +86,31 @@ export interface Website { export type WebsiteListResponse = LASListResponse; /** @public */ -export enum FormFactor { - Mobile = 'mobile', - Desktop = 'desktop', -} +export type FormFactor = 'mobile' | 'desktop'; /** @public */ -export type ScreenEmulation = { - mobile: boolean; - width: number; - height: number; - deviceScaleFactor: number; - disabled: boolean; -} | null; +export type LighthouseConfigSettings = { + // For lighthouse 7+ + formFactor: FormFactor; + screenEmulation: + | { + mobile: boolean; + width: number; + height: number; + deviceScaleFactor: number; + disabled: boolean; + } + | undefined; + // For lighthouse before 7 + emulatedFormFactor: FormFactor; +}; /** @public */ export interface TriggerAuditPayload { url: string; options: { lighthouseConfig: { - settings: { - // For lighthouse 7+ - formFactor: FormFactor; - screenEmulation: ScreenEmulation; - // For lighthouse before 7 - emulatedFormFactor: FormFactor; - }; + settings: LighthouseConfigSettings; }; }; } diff --git a/plugins/lighthouse/src/components/CreateAudit/index.test.tsx b/plugins/lighthouse/src/components/CreateAudit/index.test.tsx index 59847b516e..60a7bc1301 100644 --- a/plugins/lighthouse/src/components/CreateAudit/index.test.tsx +++ b/plugins/lighthouse/src/components/CreateAudit/index.test.tsx @@ -110,6 +110,88 @@ describe('CreateAudit', () => { }); }); + describe('when creating the audit', () => { + it('sends the correct payload for mobile', async () => { + let triggerAuditPayload: {} | undefined = undefined; + server.use( + rest.post('http://lighthouse/v1/audits', async (req, res, ctx) => { + triggerAuditPayload = await req.json(); + return res(ctx.json(createAuditResponse)); + }), + ); + + const rendered = render( + wrapInTestApp( + + + , + ), + ); + + fireEvent.change(rendered.getByLabelText(/URL/), { + target: { value: 'https://spotify.com' }, + }); + fireEvent.click(rendered.getByText(/Create Audit/)); + + await waitFor(() => + expect(triggerAuditPayload).toMatchObject({ + options: { + lighthouseConfig: { + settings: { formFactor: 'mobile', emulatedFormFactor: 'mobile' }, + }, + }, + url: 'https://spotify.com', + }), + ); + }); + + it('sends the correct payload for desktop', async () => { + let triggerAuditPayload: {} | undefined = undefined; + server.use( + rest.post('http://lighthouse/v1/audits', async (req, res, ctx) => { + triggerAuditPayload = await req.json(); + return res(ctx.json(createAuditResponse)); + }), + ); + + const rendered = render( + wrapInTestApp( + + + , + ), + ); + + fireEvent.change(rendered.getByLabelText(/URL/), { + target: { value: 'https://spotify.com' }, + }); + fireEvent.mouseDown(rendered.getByText(/Mobile/)); + fireEvent.click(rendered.getByText(/Desktop/)); + fireEvent.click(rendered.getByText(/Create Audit/)); + + await waitFor(() => + expect(triggerAuditPayload).toMatchObject({ + options: { + lighthouseConfig: { + settings: { + formFactor: 'desktop', + screenEmulation: { + mobile: false, + width: 1350, + height: 940, + deviceScaleFactor: 1, + disabled: false, + }, + emulatedFormFactor: 'desktop', + }, + }, + }, + url: 'https://spotify.com', + }), + ); + }); + }); + describe('when the audit is successfully created', () => { it('triggers a location change to the table', async () => { useNavigate.mockClear(); diff --git a/plugins/lighthouse/src/components/CreateAudit/index.tsx b/plugins/lighthouse/src/components/CreateAudit/index.tsx index 2ee5a675b6..8ed7dc8996 100644 --- a/plugins/lighthouse/src/components/CreateAudit/index.tsx +++ b/plugins/lighthouse/src/components/CreateAudit/index.tsx @@ -25,7 +25,11 @@ import { } from '@material-ui/core'; import React, { useCallback, useState } from 'react'; import { useNavigate } from 'react-router-dom'; -import { FormFactor, lighthouseApiRef, ScreenEmulation } from '../../api'; +import { + FormFactor, + lighthouseApiRef, + LighthouseConfigSettings, +} from '../../api'; import { useQuery } from '../../utils'; import LighthouseSupportButton from '../SupportButton'; @@ -65,9 +69,12 @@ const useStyles = makeStyles(theme => ({ }, })); -const formFactorToScreenEmulationMap: Record = { +const formFactorToScreenEmulationMap: Record< + FormFactor, + LighthouseConfigSettings['screenEmulation'] +> = { // the default is mobile, so no need to override - mobile: null, + mobile: undefined, // Values from lighthouse's cli "desktop" preset // https://github.com/GoogleChrome/lighthouse/blob/a6738e0033e7e5ca308b97c1c36f298b7d399402/lighthouse-core/config/constants.js#L71-L77 desktop: { @@ -87,7 +94,7 @@ export const CreateAuditContent = () => { const navigate = useNavigate(); const [submitting, setSubmitting] = useState(false); const [url, setUrl] = useState(query.get('url') || ''); - const [formFactor, setFormFactor] = useState(FormFactor.Mobile); + const [formFactor, setFormFactor] = useState('mobile'); const triggerAudit = useCallback(async (): Promise => { setSubmitting(true); From bd74fe12c0356aebf7e6993c6afc79f65336dab8 Mon Sep 17 00:00:00 2001 From: Kyle Leonhard Date: Wed, 14 Dec 2022 09:40:09 -0800 Subject: [PATCH 103/237] Update .changeset/healthy-shrimps-notice.md Co-authored-by: Johan Haals Signed-off-by: Kyle Leonhard --- .changeset/healthy-shrimps-notice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/healthy-shrimps-notice.md b/.changeset/healthy-shrimps-notice.md index cca15de504..2b06935d64 100644 --- a/.changeset/healthy-shrimps-notice.md +++ b/.changeset/healthy-shrimps-notice.md @@ -2,4 +2,4 @@ '@backstage/plugin-scaffolder-backend': minor --- -Add option to require Github conversation resolution +Added `requireConversationResolution` template option to `github:repo:create`, `github:repo:push` and `publish:github` From e0e49bb0b23d2e48bc40ba90d863852f6bcd413a Mon Sep 17 00:00:00 2001 From: Kyle Leonhard Date: Wed, 14 Dec 2022 09:55:45 -0800 Subject: [PATCH 104/237] Add tests Signed-off-by: Kyle Leonhard --- .../builtin/github/githubRepoPush.test.ts | 69 ++++++++++++++++++ .../actions/builtin/publish/github.test.ts | 71 +++++++++++++++++++ 2 files changed, 140 insertions(+) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts index e7afc26fb8..f8b45093e6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts @@ -586,4 +586,73 @@ describe('github:repo:push', () => { dismissStaleReviews: false, }); }); + + it('should call enableBranchProtectionOnDefaultRepoBranch with the correct values of requiredConversationResolution', async () => { + mockOctokit.rest.repos.get.mockResolvedValue({ + data: { + clone_url: 'https://github.com/clone/url.git', + html_url: 'https://github.com/html/url', + }, + }); + + await action.handler(mockContext); + + expect(enableBranchProtectionOnDefaultRepoBranch).toHaveBeenCalledWith({ + owner: 'owner', + client: mockOctokit, + repoName: 'repository', + logger: mockContext.logger, + defaultBranch: 'master', + requireCodeOwnerReviews: false, + requiredStatusCheckContexts: [], + requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, + enforceAdmins: true, + dismissStaleReviews: false, + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + requiredConversationResolution: true, + }, + }); + + expect(enableBranchProtectionOnDefaultRepoBranch).toHaveBeenCalledWith({ + owner: 'owner', + client: mockOctokit, + repoName: 'repository', + logger: mockContext.logger, + defaultBranch: 'master', + requireCodeOwnerReviews: false, + requiredStatusCheckContexts: [], + requireBranchesToBeUpToDate: true, + requiredConversationResolution: true, + enforceAdmins: true, + dismissStaleReviews: false, + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + requiredConversationResolution: false, + }, + }); + + expect(enableBranchProtectionOnDefaultRepoBranch).toHaveBeenCalledWith({ + owner: 'owner', + client: mockOctokit, + repoName: 'repository', + logger: mockContext.logger, + defaultBranch: 'master', + requireCodeOwnerReviews: false, + requiredStatusCheckContexts: [], + requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, + enforceAdmins: true, + dismissStaleReviews: false, + }); + }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts index 99a8753452..fa3b3c9b45 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts @@ -1001,6 +1001,77 @@ describe('publish:github', () => { }, }); + expect(enableBranchProtectionOnDefaultRepoBranch).toHaveBeenCalledWith({ + owner: 'owner', + client: mockOctokit, + repoName: 'repo', + logger: mockContext.logger, + defaultBranch: 'master', + requireCodeOwnerReviews: false, + requiredStatusCheckContexts: [], + requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, + enforceAdmins: true, + dismissStaleReviews: false, + }); + }); + it('should call enableBranchProtectionOnDefaultRepoBranch with the correct values of requiredConversationResolution', async () => { + mockOctokit.rest.users.getByUsername.mockResolvedValue({ + data: { type: 'User' }, + }); + + mockOctokit.rest.repos.createForAuthenticatedUser.mockResolvedValue({ + data: { + name: 'repo', + }, + }); + + await action.handler(mockContext); + + expect(enableBranchProtectionOnDefaultRepoBranch).toHaveBeenCalledWith({ + owner: 'owner', + client: mockOctokit, + repoName: 'repo', + logger: mockContext.logger, + defaultBranch: 'master', + requireCodeOwnerReviews: false, + requiredStatusCheckContexts: [], + requireBranchesToBeUpToDate: true, + requiredConversationResolution: false, + enforceAdmins: true, + dismissStaleReviews: false, + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + requiredConversationResolution: true, + }, + }); + + expect(enableBranchProtectionOnDefaultRepoBranch).toHaveBeenCalledWith({ + owner: 'owner', + client: mockOctokit, + repoName: 'repo', + logger: mockContext.logger, + defaultBranch: 'master', + requireCodeOwnerReviews: false, + requiredStatusCheckContexts: [], + requireBranchesToBeUpToDate: true, + requiredConversationResolution: true, + enforceAdmins: true, + dismissStaleReviews: false, + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + requiredConversationResolution: false, + }, + }); + expect(enableBranchProtectionOnDefaultRepoBranch).toHaveBeenCalledWith({ owner: 'owner', client: mockOctokit, From ac6cc9f7bdd803c47ec8ee1be84fea7de1b5ce7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 14 Dec 2022 21:56:04 +0100 Subject: [PATCH 105/237] Removed circular import in the errors package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/honest-spoons-design.md | 5 +++++ packages/errors/src/errors/CustomErrorBase.ts | 2 +- packages/errors/src/errors/ResponseError.ts | 2 +- packages/errors/src/serialization/error.ts | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 .changeset/honest-spoons-design.md diff --git a/.changeset/honest-spoons-design.md b/.changeset/honest-spoons-design.md new file mode 100644 index 0000000000..81e0740366 --- /dev/null +++ b/.changeset/honest-spoons-design.md @@ -0,0 +1,5 @@ +--- +'@backstage/errors': patch +--- + +Removed a circular import diff --git a/packages/errors/src/errors/CustomErrorBase.ts b/packages/errors/src/errors/CustomErrorBase.ts index 786e10ef23..b831d4eea3 100644 --- a/packages/errors/src/errors/CustomErrorBase.ts +++ b/packages/errors/src/errors/CustomErrorBase.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { stringifyError } from '../serialization'; +import { stringifyError } from '../serialization/error'; import { isError } from './assertion'; /** diff --git a/packages/errors/src/errors/ResponseError.ts b/packages/errors/src/errors/ResponseError.ts index a26891ef6a..f8ba0a1ba1 100644 --- a/packages/errors/src/errors/ResponseError.ts +++ b/packages/errors/src/errors/ResponseError.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { deserializeError } from '../serialization'; +import { deserializeError } from '../serialization/error'; import { ErrorResponseBody, parseErrorResponseBody, diff --git a/packages/errors/src/serialization/error.ts b/packages/errors/src/serialization/error.ts index 8d031949bf..e48309f1e7 100644 --- a/packages/errors/src/serialization/error.ts +++ b/packages/errors/src/serialization/error.ts @@ -19,7 +19,7 @@ import { deserializeError as deserializeErrorInternal, serializeError as serializeErrorInternal, } from 'serialize-error'; -import { isError } from '../errors'; +import { isError } from '../errors/assertion'; /** * The serialized form of an Error. From ce724cf5aefc4fd9811c3657fe9808cc314c381b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 14 Dec 2022 20:59:16 +0000 Subject: [PATCH 106/237] Update dependency css-loader to v6.7.3 Signed-off-by: Renovate Bot --- yarn.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/yarn.lock b/yarn.lock index 2048c5ff5e..f3ed5b9187 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19620,11 +19620,11 @@ __metadata: linkType: hard "css-loader@npm:^6.5.1": - version: 6.7.2 - resolution: "css-loader@npm:6.7.2" + version: 6.7.3 + resolution: "css-loader@npm:6.7.3" dependencies: icss-utils: ^5.1.0 - postcss: ^8.4.18 + postcss: ^8.4.19 postcss-modules-extract-imports: ^3.0.0 postcss-modules-local-by-default: ^4.0.0 postcss-modules-scope: ^3.0.0 @@ -19633,7 +19633,7 @@ __metadata: semver: ^7.3.8 peerDependencies: webpack: ^5.0.0 - checksum: f3c980cc9c033a02e60df7e5a2f33a1e8c2c3dd552f017485d2d81b383be623ae8c4189404e7a4a7403b52744683ae4b516def0f7ccf125c2b198cb647e46543 + checksum: 473cc32b6c837c2848e2051ad1ba331c1457449f47442e75a8c480d9891451434ada241f7e3de2347e57de17fcd84610b3bcfc4a9da41102cdaedd1e17902d31 languageName: node linkType: hard @@ -32145,7 +32145,7 @@ __metadata: languageName: node linkType: hard -"postcss@npm:^8.1.0, postcss@npm:^8.4.18": +"postcss@npm:^8.1.0, postcss@npm:^8.4.19": version: 8.4.20 resolution: "postcss@npm:8.4.20" dependencies: From 2b555e3b839f58b2883df80079b3e04205ec1b03 Mon Sep 17 00:00:00 2001 From: Magnus Persson Date: Tue, 22 Nov 2022 11:13:15 +0100 Subject: [PATCH 107/237] Add @backstage/plugin-sonarqube-react Signed-off-by: Magnus Persson --- plugins/sonarqube-react/.eslintrc.js | 1 + plugins/sonarqube-react/package.json | 48 ++++++++ .../src/api/SonarQubeApi.ts | 32 +++++- plugins/sonarqube-react/src/api/index.ts | 24 ++++ .../sonarqube-react/src/components/index.ts | 21 ++++ .../components/isSonarQubeAvailable.test.ts | 58 ++++++++++ .../src/components/isSonarQubeAvailable.ts | 24 ++++ plugins/sonarqube-react/src/hooks/index.ts | 17 +++ .../src/hooks/useProjectInfo.test.ts | 108 ++++++++++++++++++ .../src/hooks/useProjectInfo.ts | 59 ++++++++++ plugins/sonarqube-react/src/index.ts | 19 +++ 11 files changed, 409 insertions(+), 2 deletions(-) create mode 100644 plugins/sonarqube-react/.eslintrc.js create mode 100644 plugins/sonarqube-react/package.json rename plugins/{sonarqube => sonarqube-react}/src/api/SonarQubeApi.ts (67%) create mode 100644 plugins/sonarqube-react/src/api/index.ts create mode 100644 plugins/sonarqube-react/src/components/index.ts create mode 100644 plugins/sonarqube-react/src/components/isSonarQubeAvailable.test.ts create mode 100644 plugins/sonarqube-react/src/components/isSonarQubeAvailable.ts create mode 100644 plugins/sonarqube-react/src/hooks/index.ts create mode 100644 plugins/sonarqube-react/src/hooks/useProjectInfo.test.ts create mode 100644 plugins/sonarqube-react/src/hooks/useProjectInfo.ts create mode 100644 plugins/sonarqube-react/src/index.ts diff --git a/plugins/sonarqube-react/.eslintrc.js b/plugins/sonarqube-react/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/sonarqube-react/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/sonarqube-react/package.json b/plugins/sonarqube-react/package.json new file mode 100644 index 0000000000..99e794e6cd --- /dev/null +++ b/plugins/sonarqube-react/package.json @@ -0,0 +1,48 @@ +{ + "name": "@backstage/plugin-sonarqube-react", + "version": "0.0.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.esm.js", + "types": "dist/index.d.ts", + "alphaTypes": "dist/index.alpha.d.ts" + }, + "backstage": { + "role": "web-library" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/sonarqube-react" + }, + "keywords": [ + "backstage" + ], + "scripts": { + "build": "backstage-cli package build --experimental-type-build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "clean": "backstage-cli package clean", + "start": "backstage-cli package start" + }, + "dependencies": { + "@backstage/catalog-model": "workspace:^", + "@backstage/core-plugin-api": "workspace:^" + }, + "peerDependencies": { + "react": "^16.13.1 || ^17.0.0" + }, + "devDependencies": { + "@backstage/cli": "workspace:^" + }, + "files": [ + "dist", + "alpha" + ] +} diff --git a/plugins/sonarqube/src/api/SonarQubeApi.ts b/plugins/sonarqube-react/src/api/SonarQubeApi.ts similarity index 67% rename from plugins/sonarqube/src/api/SonarQubeApi.ts rename to plugins/sonarqube-react/src/api/SonarQubeApi.ts index 9dbb6331dd..232ca535a9 100644 --- a/plugins/sonarqube/src/api/SonarQubeApi.ts +++ b/plugins/sonarqube-react/src/api/SonarQubeApi.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,36 @@ * limitations under the License. */ -import { MetricKey, SonarUrlProcessorFunc } from './types'; import { createApiRef } from '@backstage/core-plugin-api'; +export type MetricKey = + // alert status + | 'alert_status' + + // bugs and rating (-> reliability) + | 'bugs' + | 'reliability_rating' + + // vulnerabilities and rating (-> security) + | 'vulnerabilities' + | 'security_rating' + + // code smells and rating (-> maintainability) + | 'code_smells' + | 'sqale_rating' + + // security hotspots + | 'security_hotspots_reviewed' + | 'security_review_rating' + + // coverage + | 'coverage' + + // duplicated lines + | 'duplicated_lines_density'; + +export type SonarUrlProcessorFunc = (identifier: string) => string; + /** * Define a type to make sure that all metrics are used */ @@ -37,6 +64,7 @@ export const sonarQubeApiRef = createApiRef({ id: 'plugin.sonarqube.service', }); +/** @alpha */ export type SonarQubeApi = { getFindingSummary(options: { componentKey?: string; diff --git a/plugins/sonarqube-react/src/api/index.ts b/plugins/sonarqube-react/src/api/index.ts new file mode 100644 index 0000000000..35ebdacabe --- /dev/null +++ b/plugins/sonarqube-react/src/api/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { sonarQubeApiRef } from './SonarQubeApi'; +export type { + Metrics, + MetricKey, + SonarQubeApi, + FindingSummary, + SonarUrlProcessorFunc, +} from './SonarQubeApi'; diff --git a/plugins/sonarqube-react/src/components/index.ts b/plugins/sonarqube-react/src/components/index.ts new file mode 100644 index 0000000000..a000986306 --- /dev/null +++ b/plugins/sonarqube-react/src/components/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { + isSonarQubeAvailable, + SONARQUBE_PROJECT_KEY_ANNOTATION, + SONARQUBE_PROJECT_INSTANCE_SEPARATOR, +} from './isSonarQubeAvailable'; diff --git a/plugins/sonarqube-react/src/components/isSonarQubeAvailable.test.ts b/plugins/sonarqube-react/src/components/isSonarQubeAvailable.test.ts new file mode 100644 index 0000000000..9053b1a347 --- /dev/null +++ b/plugins/sonarqube-react/src/components/isSonarQubeAvailable.test.ts @@ -0,0 +1,58 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { + isSonarQubeAvailable, + SONARQUBE_PROJECT_KEY_ANNOTATION, +} from './isSonarQubeAvailable'; + +const createDummyEntity = (sonarqubeAnnotationValue: string): Entity => { + return { + apiVersion: '', + kind: '', + metadata: { + name: 'dummy', + annotations: { + [SONARQUBE_PROJECT_KEY_ANNOTATION]: sonarqubeAnnotationValue, + }, + }, + }; +}; + +describe('isSonarQubeAvailable', () => { + it('returns true if sonarqube annotation defined', () => { + const entity = createDummyEntity('dummy'); + expect(isSonarQubeAvailable(entity)).toBe(true); + }); + + it('returns false if sonarqube annotation empty', () => { + const entity = createDummyEntity(''); + expect(isSonarQubeAvailable(entity)).toBe(false); + }); + + it('returns false if sonarqube annotation not defined', () => { + const entity = { + apiVersion: '', + kind: '', + metadata: { + name: 'dummy', + annotations: {}, + }, + }; + expect(isSonarQubeAvailable(entity)).toBe(false); + }); +}); diff --git a/plugins/sonarqube-react/src/components/isSonarQubeAvailable.ts b/plugins/sonarqube-react/src/components/isSonarQubeAvailable.ts new file mode 100644 index 0000000000..b8504809c3 --- /dev/null +++ b/plugins/sonarqube-react/src/components/isSonarQubeAvailable.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; + +export const SONARQUBE_PROJECT_KEY_ANNOTATION = 'sonarqube.org/project-key'; +export const SONARQUBE_PROJECT_INSTANCE_SEPARATOR = '/'; + +/** @public */ +export const isSonarQubeAvailable = (entity: Entity) => + Boolean(entity.metadata.annotations?.[SONARQUBE_PROJECT_KEY_ANNOTATION]); diff --git a/plugins/sonarqube-react/src/hooks/index.ts b/plugins/sonarqube-react/src/hooks/index.ts new file mode 100644 index 0000000000..16892a5e3d --- /dev/null +++ b/plugins/sonarqube-react/src/hooks/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { useProjectInfo } from './useProjectInfo'; diff --git a/plugins/sonarqube-react/src/hooks/useProjectInfo.test.ts b/plugins/sonarqube-react/src/hooks/useProjectInfo.test.ts new file mode 100644 index 0000000000..2babb4cd42 --- /dev/null +++ b/plugins/sonarqube-react/src/hooks/useProjectInfo.test.ts @@ -0,0 +1,108 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { + SONARQUBE_PROJECT_INSTANCE_SEPARATOR, + SONARQUBE_PROJECT_KEY_ANNOTATION, +} from '../components'; +import { useProjectInfo } from './useProjectInfo'; + +const createDummyEntity = (sonarqubeAnnotationValue: string): Entity => { + return { + apiVersion: '', + kind: '', + metadata: { + name: 'dummy', + annotations: { + [SONARQUBE_PROJECT_KEY_ANNOTATION]: sonarqubeAnnotationValue, + }, + }, + }; +}; + +describe('useProjectInfo', () => { + const DUMMY_INSTANCE = 'dummyInstance'; + const DUMMY_KEY = 'dummyKey'; + + it('parse annotation with key and instance', () => { + const entity = createDummyEntity( + DUMMY_INSTANCE + SONARQUBE_PROJECT_INSTANCE_SEPARATOR + DUMMY_KEY, + ); + expect(useProjectInfo(entity)).toEqual({ + projectInstance: DUMMY_INSTANCE, + projectKey: DUMMY_KEY, + }); + }); + + it('parse annotation with instance, tenant/project-key', () => { + const DUMMY_KEY_WITH_TENANT = 'dummy-tenant/dummyKey'; + const entity = createDummyEntity( + DUMMY_INSTANCE + + SONARQUBE_PROJECT_INSTANCE_SEPARATOR + + DUMMY_KEY_WITH_TENANT, + ); + expect(useProjectInfo(entity)).toEqual({ + projectInstance: DUMMY_INSTANCE, + projectKey: DUMMY_KEY_WITH_TENANT, + }); + }); + + it('parse annotation with instance, tenant:project-key', () => { + const DUMMY_KEY_WITH_TENANT = 'dummy-tenant:dummyKey'; + const entity = createDummyEntity( + DUMMY_INSTANCE + + SONARQUBE_PROJECT_INSTANCE_SEPARATOR + + DUMMY_KEY_WITH_TENANT, + ); + expect(useProjectInfo(entity)).toEqual({ + projectInstance: DUMMY_INSTANCE, + projectKey: DUMMY_KEY_WITH_TENANT, + }); + }); + + // compatibility with previous mono-instance sonarqube config + it('parse annotation with only key', () => { + const entity = createDummyEntity(DUMMY_KEY); + expect(useProjectInfo(entity)).toEqual({ + projectInstance: undefined, + projectKey: DUMMY_KEY, + }); + }); + + it('handle empty annotation', () => { + const entity = createDummyEntity(''); + expect(useProjectInfo(entity)).toEqual({ + projectInstance: undefined, + projectKey: undefined, + }); + }); + + it('handle non-existent annotation', () => { + const entity = { + apiVersion: '', + kind: '', + metadata: { + name: 'dummy', + annotations: {}, + }, + }; + expect(useProjectInfo(entity)).toEqual({ + projectInstance: undefined, + projectKey: undefined, + }); + }); +}); diff --git a/plugins/sonarqube-react/src/hooks/useProjectInfo.ts b/plugins/sonarqube-react/src/hooks/useProjectInfo.ts new file mode 100644 index 0000000000..61c8baed98 --- /dev/null +++ b/plugins/sonarqube-react/src/hooks/useProjectInfo.ts @@ -0,0 +1,59 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { + SONARQUBE_PROJECT_INSTANCE_SEPARATOR, + SONARQUBE_PROJECT_KEY_ANNOTATION, +} from '../components'; + +/** + * + * Try to parse sonarqube information from an entity. + * + * If part or all info are not found, they will default to undefined + * + * @alpha + * @param entity entity to find the sonarqube information from. + * @return a ProjectInfo properly populated. + */ +export const useProjectInfo = ( + entity: Entity, +): { + projectInstance: string | undefined; + projectKey: string | undefined; +} => { + let projectInstance = undefined; + let projectKey = undefined; + const annotation = + entity?.metadata.annotations?.[SONARQUBE_PROJECT_KEY_ANNOTATION]; + if (annotation) { + const instanceSeparatorIndex = annotation.indexOf( + SONARQUBE_PROJECT_INSTANCE_SEPARATOR, + ); + if (instanceSeparatorIndex > -1) { + // Examples: + // instanceA/projectA -> projectInstance = "instanceA" & projectKey = "projectA" + // instanceA/tenantA:projectA -> projectInstance = "instanceA" & projectKey = "tenantA:projectA" + // instanceA/tenantA/projectA -> projectInstance = "instanceA" & projectKey = "tenantA/projectA" + projectInstance = annotation.substring(0, instanceSeparatorIndex); + projectKey = annotation.substring(instanceSeparatorIndex + 1); + } else { + projectKey = annotation; + } + } + return { projectInstance, projectKey }; +}; diff --git a/plugins/sonarqube-react/src/index.ts b/plugins/sonarqube-react/src/index.ts new file mode 100644 index 0000000000..fe04a0459f --- /dev/null +++ b/plugins/sonarqube-react/src/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './api'; +export * from './components'; +export * from './hooks'; From b3a304e78e6a2276e25fd763f21a3a561c275b96 Mon Sep 17 00:00:00 2001 From: Magnus Persson Date: Tue, 22 Nov 2022 11:37:12 +0100 Subject: [PATCH 108/237] Reference @backstage/plugin-sonarqube-react from @backstage/plugin-sonarqube Signed-off-by: Magnus Persson --- plugins/sonarqube/dev/index.tsx | 8 +- plugins/sonarqube/package.json | 1 + .../sonarqube/src/api/SonarQubeClient.test.ts | 3 +- plugins/sonarqube/src/api/SonarQubeClient.ts | 6 +- plugins/sonarqube/src/api/index.ts | 2 - plugins/sonarqube/src/api/types.ts | 30 +--- .../SonarQubeCard/SonarQubeCard.tsx | 10 +- .../SonarQubeContentPage.test.tsx | 5 +- .../SonarQubeContentPage.tsx | 4 +- plugins/sonarqube/src/components/index.ts | 4 - .../src/components/useProjectKey.test.ts | 132 ------------------ .../sonarqube/src/components/useProjectKey.ts | 61 -------- plugins/sonarqube/src/plugin.ts | 3 +- yarn.lock | 13 ++ 14 files changed, 41 insertions(+), 241 deletions(-) delete mode 100644 plugins/sonarqube/src/components/useProjectKey.test.ts delete mode 100644 plugins/sonarqube/src/components/useProjectKey.ts diff --git a/plugins/sonarqube/dev/index.tsx b/plugins/sonarqube/dev/index.tsx index bba859195f..a34c4d4e82 100644 --- a/plugins/sonarqube/dev/index.tsx +++ b/plugins/sonarqube/dev/index.tsx @@ -19,9 +19,13 @@ import { createDevApp, EntityGridItem } from '@backstage/dev-utils'; import { Grid } from '@material-ui/core'; import React from 'react'; import { EntitySonarQubeCard, sonarQubePlugin } from '../src'; -import { FindingSummary, SonarQubeApi, sonarQubeApiRef } from '../src/api'; -import { SONARQUBE_PROJECT_KEY_ANNOTATION } from '../src/components/useProjectKey'; import { Content, Header, Page } from '@backstage/core-components'; +import { + FindingSummary, + SONARQUBE_PROJECT_KEY_ANNOTATION, + SonarQubeApi, + sonarQubeApiRef, +} from '@backstage/plugin-sonarqube-react'; const entity = (name?: string) => ({ diff --git a/plugins/sonarqube/package.json b/plugins/sonarqube/package.json index ad6df1db8c..a5465eee78 100644 --- a/plugins/sonarqube/package.json +++ b/plugins/sonarqube/package.json @@ -38,6 +38,7 @@ "@backstage/core-components": "workspace:^", "@backstage/core-plugin-api": "workspace:^", "@backstage/plugin-catalog-react": "workspace:^", + "@backstage/plugin-sonarqube-react": "workspace:^", "@backstage/theme": "workspace:^", "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", diff --git a/plugins/sonarqube/src/api/SonarQubeClient.test.ts b/plugins/sonarqube/src/api/SonarQubeClient.test.ts index dc5d7f077e..592f08457f 100644 --- a/plugins/sonarqube/src/api/SonarQubeClient.test.ts +++ b/plugins/sonarqube/src/api/SonarQubeClient.test.ts @@ -17,10 +17,11 @@ import { setupRequestMockHandlers } from '@backstage/test-utils'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import { FindingSummary, SonarQubeClient } from './index'; +import { SonarQubeClient } from './index'; import { InstanceUrlWrapper, FindingsWrapper } from './types'; import { UrlPatternDiscovery } from '@backstage/core-app-api'; import { IdentityApi } from '@backstage/core-plugin-api'; +import { FindingSummary } from '@backstage/plugin-sonarqube-react'; const server = setupServer(); diff --git a/plugins/sonarqube/src/api/SonarQubeClient.ts b/plugins/sonarqube/src/api/SonarQubeClient.ts index 9ec40e8420..cf893ccef5 100644 --- a/plugins/sonarqube/src/api/SonarQubeClient.ts +++ b/plugins/sonarqube/src/api/SonarQubeClient.ts @@ -15,7 +15,11 @@ */ import fetch from 'cross-fetch'; -import { FindingSummary, Metrics, SonarQubeApi } from './SonarQubeApi'; +import { + FindingSummary, + Metrics, + SonarQubeApi, +} from '@backstage/plugin-sonarqube-react'; import { InstanceUrlWrapper, FindingsWrapper } from './types'; import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api'; diff --git a/plugins/sonarqube/src/api/index.ts b/plugins/sonarqube/src/api/index.ts index 8c418016cd..ddf355af33 100644 --- a/plugins/sonarqube/src/api/index.ts +++ b/plugins/sonarqube/src/api/index.ts @@ -14,6 +14,4 @@ * limitations under the License. */ -export type { Metrics, FindingSummary, SonarQubeApi } from './SonarQubeApi'; -export { sonarQubeApiRef } from './SonarQubeApi'; export { SonarQubeClient } from './SonarQubeClient'; diff --git a/plugins/sonarqube/src/api/types.ts b/plugins/sonarqube/src/api/types.ts index 348986149d..48c7c41c03 100644 --- a/plugins/sonarqube/src/api/types.ts +++ b/plugins/sonarqube/src/api/types.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { MetricKey } from '@backstage/plugin-sonarqube-react'; + export interface InstanceUrlWrapper { instanceUrl: string; } @@ -23,35 +25,7 @@ export interface FindingsWrapper { measures: Measure[]; } -export type MetricKey = - // alert status - | 'alert_status' - - // bugs and rating (-> reliability) - | 'bugs' - | 'reliability_rating' - - // vulnerabilities and rating (-> security) - | 'vulnerabilities' - | 'security_rating' - - // code smells and rating (-> maintainability) - | 'code_smells' - | 'sqale_rating' - - // security hotspots - | 'security_hotspots_reviewed' - | 'security_review_rating' - - // coverage - | 'coverage' - - // duplicated lines - | 'duplicated_lines_density'; - export interface Measure { metric: MetricKey; value: string; } - -export type SonarUrlProcessorFunc = (identifier: string) => string; diff --git a/plugins/sonarqube/src/components/SonarQubeCard/SonarQubeCard.tsx b/plugins/sonarqube/src/components/SonarQubeCard/SonarQubeCard.tsx index b32a66aaa3..900c8d082c 100644 --- a/plugins/sonarqube/src/components/SonarQubeCard/SonarQubeCard.tsx +++ b/plugins/sonarqube/src/components/SonarQubeCard/SonarQubeCard.tsx @@ -15,6 +15,11 @@ */ import { useEntity } from '@backstage/plugin-catalog-react'; +import { + SONARQUBE_PROJECT_KEY_ANNOTATION, + sonarQubeApiRef, + useProjectInfo, +} from '@backstage/plugin-sonarqube-react'; import { Chip, Grid } from '@material-ui/core'; import { makeStyles } from '@material-ui/core/styles'; import BugReport from '@material-ui/icons/BugReport'; @@ -23,11 +28,6 @@ import Security from '@material-ui/icons/Security'; import SentimentVeryDissatisfied from '@material-ui/icons/SentimentVeryDissatisfied'; import React, { useMemo } from 'react'; import useAsync from 'react-use/lib/useAsync'; -import { sonarQubeApiRef } from '../../api'; -import { - SONARQUBE_PROJECT_KEY_ANNOTATION, - useProjectInfo, -} from '../useProjectKey'; import { Percentage } from './Percentage'; import { Rating } from './Rating'; import { RatingCard } from './RatingCard'; diff --git a/plugins/sonarqube/src/components/SonarQubeContentPage/SonarQubeContentPage.test.tsx b/plugins/sonarqube/src/components/SonarQubeContentPage/SonarQubeContentPage.test.tsx index 8e847d7c31..22175d14bc 100644 --- a/plugins/sonarqube/src/components/SonarQubeContentPage/SonarQubeContentPage.test.tsx +++ b/plugins/sonarqube/src/components/SonarQubeContentPage/SonarQubeContentPage.test.tsx @@ -22,8 +22,9 @@ import React from 'react'; import { isSonarQubeAvailable, SONARQUBE_PROJECT_KEY_ANNOTATION, -} from '../useProjectKey'; -import { SonarQubeApi, sonarQubeApiRef } from '../../api'; + SonarQubeApi, + sonarQubeApiRef, +} from '@backstage/plugin-sonarqube-react'; const Providers = ({ annotation, diff --git a/plugins/sonarqube/src/components/SonarQubeContentPage/SonarQubeContentPage.tsx b/plugins/sonarqube/src/components/SonarQubeContentPage/SonarQubeContentPage.tsx index 33a145b746..f577d08eeb 100644 --- a/plugins/sonarqube/src/components/SonarQubeContentPage/SonarQubeContentPage.tsx +++ b/plugins/sonarqube/src/components/SonarQubeContentPage/SonarQubeContentPage.tsx @@ -22,11 +22,11 @@ import { import { MissingAnnotationEmptyState } from '@backstage/core-components'; import { useEntity } from '@backstage/plugin-catalog-react'; import React from 'react'; +import { SonarQubeCard } from '../SonarQubeCard'; import { isSonarQubeAvailable, SONARQUBE_PROJECT_KEY_ANNOTATION, -} from '../useProjectKey'; -import { SonarQubeCard } from '../SonarQubeCard'; +} from '@backstage/plugin-sonarqube-react'; /** @public */ export type SonarQubeContentPageProps = { diff --git a/plugins/sonarqube/src/components/index.ts b/plugins/sonarqube/src/components/index.ts index 8e1d93f85b..85ccfe9621 100644 --- a/plugins/sonarqube/src/components/index.ts +++ b/plugins/sonarqube/src/components/index.ts @@ -17,7 +17,3 @@ export { SonarQubeCard } from './SonarQubeCard'; export type { DuplicationRating } from './SonarQubeCard'; export type { SonarQubeContentPageProps } from './SonarQubeContentPage'; -export { - isSonarQubeAvailable, - SONARQUBE_PROJECT_KEY_ANNOTATION, -} from './useProjectKey'; diff --git a/plugins/sonarqube/src/components/useProjectKey.test.ts b/plugins/sonarqube/src/components/useProjectKey.test.ts deleted file mode 100644 index 8686ac82de..0000000000 --- a/plugins/sonarqube/src/components/useProjectKey.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { - isSonarQubeAvailable, - SONARQUBE_PROJECT_INSTANCE_SEPARATOR, - SONARQUBE_PROJECT_KEY_ANNOTATION, - useProjectInfo, -} from './useProjectKey'; -import { Entity } from '@backstage/catalog-model'; - -const createDummyEntity = (sonarqubeAnnotationValue: string): Entity => { - return { - apiVersion: '', - kind: '', - metadata: { - name: 'dummy', - annotations: { - [SONARQUBE_PROJECT_KEY_ANNOTATION]: sonarqubeAnnotationValue, - }, - }, - }; -}; - -describe('isSonarQubeAvailable', () => { - it('returns true if sonarqube annotation defined', () => { - const entity = createDummyEntity('dummy'); - expect(isSonarQubeAvailable(entity)).toBe(true); - }); - - it('returns false if sonarqube annotation empty', () => { - const entity = createDummyEntity(''); - expect(isSonarQubeAvailable(entity)).toBe(false); - }); - - it('returns false if sonarqube annotation not defined', () => { - const entity = { - apiVersion: '', - kind: '', - metadata: { - name: 'dummy', - annotations: {}, - }, - }; - expect(isSonarQubeAvailable(entity)).toBe(false); - }); -}); - -describe('useProjectInfo', () => { - const DUMMY_INSTANCE = 'dummyInstance'; - const DUMMY_KEY = 'dummyKey'; - - it('parse annotation with key and instance', () => { - const entity = createDummyEntity( - DUMMY_INSTANCE + SONARQUBE_PROJECT_INSTANCE_SEPARATOR + DUMMY_KEY, - ); - expect(useProjectInfo(entity)).toEqual({ - projectInstance: DUMMY_INSTANCE, - projectKey: DUMMY_KEY, - }); - }); - - it('parse annotation with instance, tenant/project-key', () => { - const DUMMY_KEY_WITH_TENANT = 'dummy-tenant/dummyKey'; - const entity = createDummyEntity( - DUMMY_INSTANCE + - SONARQUBE_PROJECT_INSTANCE_SEPARATOR + - DUMMY_KEY_WITH_TENANT, - ); - expect(useProjectInfo(entity)).toEqual({ - projectInstance: DUMMY_INSTANCE, - projectKey: DUMMY_KEY_WITH_TENANT, - }); - }); - - it('parse annotation with instance, tenant:project-key', () => { - const DUMMY_KEY_WITH_TENANT = 'dummy-tenant:dummyKey'; - const entity = createDummyEntity( - DUMMY_INSTANCE + - SONARQUBE_PROJECT_INSTANCE_SEPARATOR + - DUMMY_KEY_WITH_TENANT, - ); - expect(useProjectInfo(entity)).toEqual({ - projectInstance: DUMMY_INSTANCE, - projectKey: DUMMY_KEY_WITH_TENANT, - }); - }); - - // compatibility with previous mono-instance sonarqube config - it('parse annotation with only key', () => { - const entity = createDummyEntity(DUMMY_KEY); - expect(useProjectInfo(entity)).toEqual({ - projectInstance: undefined, - projectKey: DUMMY_KEY, - }); - }); - - it('handle empty annotation', () => { - const entity = createDummyEntity(''); - expect(useProjectInfo(entity)).toEqual({ - projectInstance: undefined, - projectKey: undefined, - }); - }); - - it('handle non-existent annotation', () => { - const entity = { - apiVersion: '', - kind: '', - metadata: { - name: 'dummy', - annotations: {}, - }, - }; - expect(useProjectInfo(entity)).toEqual({ - projectInstance: undefined, - projectKey: undefined, - }); - }); -}); diff --git a/plugins/sonarqube/src/components/useProjectKey.ts b/plugins/sonarqube/src/components/useProjectKey.ts deleted file mode 100644 index 20f7f03bb3..0000000000 --- a/plugins/sonarqube/src/components/useProjectKey.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Entity } from '@backstage/catalog-model'; - -/** @public */ -export const SONARQUBE_PROJECT_KEY_ANNOTATION = 'sonarqube.org/project-key'; -export const SONARQUBE_PROJECT_INSTANCE_SEPARATOR = '/'; - -/** @public */ -export const isSonarQubeAvailable = (entity: Entity) => - Boolean(entity.metadata.annotations?.[SONARQUBE_PROJECT_KEY_ANNOTATION]); - -/** - * Try to parse sonarqube information from an entity. - * - * If part or all info are not found, they will default to undefined - * - * @param entity entity to find the sonarqube information from. - * @return a ProjectInfo properly populated. - */ -export const useProjectInfo = ( - entity: Entity, -): { - projectInstance: string | undefined; - projectKey: string | undefined; -} => { - let projectInstance = undefined; - let projectKey = undefined; - const annotation = - entity?.metadata.annotations?.[SONARQUBE_PROJECT_KEY_ANNOTATION]; - if (annotation) { - const instanceSeparatorIndex = annotation.indexOf( - SONARQUBE_PROJECT_INSTANCE_SEPARATOR, - ); - if (instanceSeparatorIndex > -1) { - // Examples: - // instanceA/projectA -> projectInstance = "instanceA" & projectKey = "projectA" - // instanceA/tenantA:projectA -> projectInstance = "instanceA" & projectKey = "tenantA:projectA" - // instanceA/tenantA/projectA -> projectInstance = "instanceA" & projectKey = "tenantA/projectA" - projectInstance = annotation.substring(0, instanceSeparatorIndex); - projectKey = annotation.substring(instanceSeparatorIndex + 1); - } else { - projectKey = annotation; - } - } - return { projectInstance, projectKey }; -}; diff --git a/plugins/sonarqube/src/plugin.ts b/plugins/sonarqube/src/plugin.ts index 5eb5ea82f3..c66d8c4e96 100644 --- a/plugins/sonarqube/src/plugin.ts +++ b/plugins/sonarqube/src/plugin.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { sonarQubeApiRef, SonarQubeClient } from './api'; +import { SonarQubeClient } from './api'; import { createApiFactory, createComponentExtension, @@ -22,6 +22,7 @@ import { discoveryApiRef, identityApiRef, } from '@backstage/core-plugin-api'; +import { sonarQubeApiRef } from '@backstage/plugin-sonarqube-react'; /** @public */ export const sonarQubePlugin = createPlugin({ diff --git a/yarn.lock b/yarn.lock index 2048c5ff5e..bcaf28da01 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7785,6 +7785,18 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-sonarqube-react@workspace:^, @backstage/plugin-sonarqube-react@workspace:plugins/sonarqube-react": + version: 0.0.0-use.local + resolution: "@backstage/plugin-sonarqube-react@workspace:plugins/sonarqube-react" + dependencies: + "@backstage/catalog-model": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/core-plugin-api": "workspace:^" + peerDependencies: + react: ^16.13.1 || ^17.0.0 + languageName: unknown + linkType: soft + "@backstage/plugin-sonarqube@workspace:plugins/sonarqube": version: 0.0.0-use.local resolution: "@backstage/plugin-sonarqube@workspace:plugins/sonarqube" @@ -7796,6 +7808,7 @@ __metadata: "@backstage/core-plugin-api": "workspace:^" "@backstage/dev-utils": "workspace:^" "@backstage/plugin-catalog-react": "workspace:^" + "@backstage/plugin-sonarqube-react": "workspace:^" "@backstage/test-utils": "workspace:^" "@backstage/theme": "workspace:^" "@material-ui/core": ^4.12.2 From 19f8ad42626ff082b4022cf6671df763dec3d8e9 Mon Sep 17 00:00:00 2001 From: Magnus Persson Date: Tue, 13 Dec 2022 22:49:54 +0100 Subject: [PATCH 109/237] Keep types but marked deprecated Signed-off-by: Magnus Persson --- plugins/sonarqube/src/api/types.ts | 13 ++++++++++++- plugins/sonarqube/src/components/index.ts | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/plugins/sonarqube/src/api/types.ts b/plugins/sonarqube/src/api/types.ts index 48c7c41c03..470d05903a 100644 --- a/plugins/sonarqube/src/api/types.ts +++ b/plugins/sonarqube/src/api/types.ts @@ -14,7 +14,8 @@ * limitations under the License. */ -import { MetricKey } from '@backstage/plugin-sonarqube-react'; +import { MetricKey as NonDeprecatedMetricKey } from '@backstage/plugin-sonarqube-react'; +import { SonarUrlProcessorFunc as NonDeprecatedSonarUrlProcessorFunc } from '@backstage/plugin-sonarqube-react'; export interface InstanceUrlWrapper { instanceUrl: string; @@ -25,7 +26,17 @@ export interface FindingsWrapper { measures: Measure[]; } +/** + * @deprecated use the same type from `@backstage/plugin-sonarqube-react` instead + */ +export type MetricKey = NonDeprecatedMetricKey; + export interface Measure { metric: MetricKey; value: string; } + +/** + * @deprecated use the same type from `@backstage/plugin-sonarqube-react` instead + */ +export type SonarUrlProcessorFunc = NonDeprecatedSonarUrlProcessorFunc; diff --git a/plugins/sonarqube/src/components/index.ts b/plugins/sonarqube/src/components/index.ts index 85ccfe9621..08af5591fd 100644 --- a/plugins/sonarqube/src/components/index.ts +++ b/plugins/sonarqube/src/components/index.ts @@ -14,6 +14,22 @@ * limitations under the License. */ +import { + isSonarQubeAvailable as NonDeprecatedIsSonarQubeAvailable, + SONARQUBE_PROJECT_KEY_ANNOTATION as NON_DEPRECATED_SONARQUBE_PROJECT_KEY_ANNOTATION, +} from '@backstage/plugin-sonarqube-react'; + export { SonarQubeCard } from './SonarQubeCard'; export type { DuplicationRating } from './SonarQubeCard'; export type { SonarQubeContentPageProps } from './SonarQubeContentPage'; + +/** + * @deprecated use the same type from `@backstage/plugin-sonarqube-react` instead + */ +export const isSonarQubeAvailable = NonDeprecatedIsSonarQubeAvailable; + +/** + * @deprecated use the same type from `@backstage/plugin-sonarqube-react` instead + */ +export const SONARQUBE_PROJECT_KEY_ANNOTATION = + NON_DEPRECATED_SONARQUBE_PROJECT_KEY_ANNOTATION; From 836b30e61ca59846db6db2fe2e450f0f66660448 Mon Sep 17 00:00:00 2001 From: Magnus Persson Date: Tue, 22 Nov 2022 12:53:57 +0100 Subject: [PATCH 110/237] Update api-report.md Signed-off-by: Magnus Persson --- plugins/sonarqube-react/api-report.md | 71 +++++++++++++++++++ .../sonarqube-react/src/api/SonarQubeApi.ts | 6 ++ .../sonarqube-react/src/components/index.ts | 1 - .../src/components/isSonarQubeAvailable.ts | 2 +- .../src/hooks/useProjectInfo.test.ts | 6 +- .../src/hooks/useProjectInfo.ts | 11 ++- plugins/sonarqube/api-report.md | 4 +- plugins/sonarqube/src/components/index.ts | 4 ++ 8 files changed, 91 insertions(+), 14 deletions(-) create mode 100644 plugins/sonarqube-react/api-report.md diff --git a/plugins/sonarqube-react/api-report.md b/plugins/sonarqube-react/api-report.md new file mode 100644 index 0000000000..be84c45cce --- /dev/null +++ b/plugins/sonarqube-react/api-report.md @@ -0,0 +1,71 @@ +## API Report File for "@backstage/plugin-sonarqube-react" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { ApiRef } from '@backstage/core-plugin-api'; +import { Entity } from '@backstage/catalog-model'; + +// @alpha (undocumented) +export interface FindingSummary { + // (undocumented) + getComponentMeasuresUrl: SonarUrlProcessorFunc; + // (undocumented) + getIssuesUrl: SonarUrlProcessorFunc; + // (undocumented) + getSecurityHotspotsUrl: () => string; + // (undocumented) + lastAnalysis: string; + // (undocumented) + metrics: Metrics; + // (undocumented) + projectUrl: string; +} + +// @public (undocumented) +export const isSonarQubeAvailable: (entity: Entity) => boolean; + +// @alpha (undocumented) +export type MetricKey = + | 'alert_status' + | 'bugs' + | 'reliability_rating' + | 'vulnerabilities' + | 'security_rating' + | 'code_smells' + | 'sqale_rating' + | 'security_hotspots_reviewed' + | 'security_review_rating' + | 'coverage' + | 'duplicated_lines_density'; + +// @alpha +export type Metrics = { + [key in MetricKey]: string | undefined; +}; + +// @public (undocumented) +export const SONARQUBE_PROJECT_KEY_ANNOTATION = 'sonarqube.org/project-key'; + +// @alpha (undocumented) +export type SonarQubeApi = { + getFindingSummary(options: { + componentKey?: string; + projectInstance?: string; + }): Promise; +}; + +// @alpha (undocumented) +export const sonarQubeApiRef: ApiRef; + +// @alpha (undocumented) +export type SonarUrlProcessorFunc = (identifier: string) => string; + +// @alpha +export const useProjectInfo: (entity: Entity) => { + projectInstance: string | undefined; + projectKey: string | undefined; +}; + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/sonarqube-react/src/api/SonarQubeApi.ts b/plugins/sonarqube-react/src/api/SonarQubeApi.ts index 232ca535a9..ee6b1bb6a3 100644 --- a/plugins/sonarqube-react/src/api/SonarQubeApi.ts +++ b/plugins/sonarqube-react/src/api/SonarQubeApi.ts @@ -16,6 +16,7 @@ import { createApiRef } from '@backstage/core-plugin-api'; +/** @alpha */ export type MetricKey = // alert status | 'alert_status' @@ -42,15 +43,19 @@ export type MetricKey = // duplicated lines | 'duplicated_lines_density'; +/** @alpha */ export type SonarUrlProcessorFunc = (identifier: string) => string; /** + * @alpha + * * Define a type to make sure that all metrics are used */ export type Metrics = { [key in MetricKey]: string | undefined; }; +/** @alpha */ export interface FindingSummary { lastAnalysis: string; metrics: Metrics; @@ -60,6 +65,7 @@ export interface FindingSummary { getSecurityHotspotsUrl: () => string; } +/** @alpha */ export const sonarQubeApiRef = createApiRef({ id: 'plugin.sonarqube.service', }); diff --git a/plugins/sonarqube-react/src/components/index.ts b/plugins/sonarqube-react/src/components/index.ts index a000986306..44c63fe052 100644 --- a/plugins/sonarqube-react/src/components/index.ts +++ b/plugins/sonarqube-react/src/components/index.ts @@ -17,5 +17,4 @@ export { isSonarQubeAvailable, SONARQUBE_PROJECT_KEY_ANNOTATION, - SONARQUBE_PROJECT_INSTANCE_SEPARATOR, } from './isSonarQubeAvailable'; diff --git a/plugins/sonarqube-react/src/components/isSonarQubeAvailable.ts b/plugins/sonarqube-react/src/components/isSonarQubeAvailable.ts index b8504809c3..d538b700f3 100644 --- a/plugins/sonarqube-react/src/components/isSonarQubeAvailable.ts +++ b/plugins/sonarqube-react/src/components/isSonarQubeAvailable.ts @@ -16,8 +16,8 @@ import { Entity } from '@backstage/catalog-model'; +/** @public */ export const SONARQUBE_PROJECT_KEY_ANNOTATION = 'sonarqube.org/project-key'; -export const SONARQUBE_PROJECT_INSTANCE_SEPARATOR = '/'; /** @public */ export const isSonarQubeAvailable = (entity: Entity) => diff --git a/plugins/sonarqube-react/src/hooks/useProjectInfo.test.ts b/plugins/sonarqube-react/src/hooks/useProjectInfo.test.ts index 2babb4cd42..db5e1f9689 100644 --- a/plugins/sonarqube-react/src/hooks/useProjectInfo.test.ts +++ b/plugins/sonarqube-react/src/hooks/useProjectInfo.test.ts @@ -15,11 +15,9 @@ */ import { Entity } from '@backstage/catalog-model'; -import { - SONARQUBE_PROJECT_INSTANCE_SEPARATOR, - SONARQUBE_PROJECT_KEY_ANNOTATION, -} from '../components'; +import { SONARQUBE_PROJECT_KEY_ANNOTATION } from '../components'; import { useProjectInfo } from './useProjectInfo'; +import { SONARQUBE_PROJECT_INSTANCE_SEPARATOR } from './useProjectInfo'; const createDummyEntity = (sonarqubeAnnotationValue: string): Entity => { return { diff --git a/plugins/sonarqube-react/src/hooks/useProjectInfo.ts b/plugins/sonarqube-react/src/hooks/useProjectInfo.ts index 61c8baed98..119e568e46 100644 --- a/plugins/sonarqube-react/src/hooks/useProjectInfo.ts +++ b/plugins/sonarqube-react/src/hooks/useProjectInfo.ts @@ -15,10 +15,9 @@ */ import { Entity } from '@backstage/catalog-model'; -import { - SONARQUBE_PROJECT_INSTANCE_SEPARATOR, - SONARQUBE_PROJECT_KEY_ANNOTATION, -} from '../components'; +import { SONARQUBE_PROJECT_KEY_ANNOTATION } from '../components'; + +export const SONARQUBE_PROJECT_INSTANCE_SEPARATOR = '/'; /** * @@ -27,8 +26,8 @@ import { * If part or all info are not found, they will default to undefined * * @alpha - * @param entity entity to find the sonarqube information from. - * @return a ProjectInfo properly populated. + * @param entity - entity to find the sonarqube information from. + * @returns a ProjectInfo properly populated. */ export const useProjectInfo = ( entity: Entity, diff --git a/plugins/sonarqube/api-report.md b/plugins/sonarqube/api-report.md index 2edf80618e..31e98a0f4a 100644 --- a/plugins/sonarqube/api-report.md +++ b/plugins/sonarqube/api-report.md @@ -26,10 +26,10 @@ export const EntitySonarQubeContentPage: ( props: SonarQubeContentPageProps, ) => JSX.Element; -// @public (undocumented) +// @public @deprecated (undocumented) export const isSonarQubeAvailable: (entity: Entity) => boolean; -// @public (undocumented) +// @public @deprecated (undocumented) export const SONARQUBE_PROJECT_KEY_ANNOTATION = 'sonarqube.org/project-key'; // @public (undocumented) diff --git a/plugins/sonarqube/src/components/index.ts b/plugins/sonarqube/src/components/index.ts index 08af5591fd..252c12c5db 100644 --- a/plugins/sonarqube/src/components/index.ts +++ b/plugins/sonarqube/src/components/index.ts @@ -24,11 +24,15 @@ export type { DuplicationRating } from './SonarQubeCard'; export type { SonarQubeContentPageProps } from './SonarQubeContentPage'; /** + * @public + * * @deprecated use the same type from `@backstage/plugin-sonarqube-react` instead */ export const isSonarQubeAvailable = NonDeprecatedIsSonarQubeAvailable; /** + * @public + * * @deprecated use the same type from `@backstage/plugin-sonarqube-react` instead */ export const SONARQUBE_PROJECT_KEY_ANNOTATION = From 6b59903bfad487f4b69717a3f04f9c3eb395de8e Mon Sep 17 00:00:00 2001 From: Magnus Persson Date: Wed, 14 Dec 2022 14:52:30 +0100 Subject: [PATCH 111/237] Add changeset Signed-off-by: Magnus Persson --- .changeset/short-pigs-juggle.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .changeset/short-pigs-juggle.md diff --git a/.changeset/short-pigs-juggle.md b/.changeset/short-pigs-juggle.md new file mode 100644 index 0000000000..c7ef43e62a --- /dev/null +++ b/.changeset/short-pigs-juggle.md @@ -0,0 +1,22 @@ +--- +'@backstage/plugin-sonarqube': minor +'@backstage/plugin-sonarqube-react': minor +--- + +Parts of plugin-sonarqube have been moved into a new plugin-sonarqube-react package. Additionally some types that were +previously internal to plugin-sonarqube have been made public and will allow access for third-parties. As the sonarqube +plugin has not yet reached 1.0 breaking changes are expected in the future. As such exports of plugin-sonarqube-react +require importing via the `/alpha` entrypoint: + +```ts +import { sonarQubeApiRef } from '@backstage/plugin-sonarqube-react/alpha'; + +const sonarQubeApi = useApi(sonarQubeApiRef); +``` + +Moved from plugin-sonarqube to plugin-sonarqube-react: + +- isSonarQubeAvailable +- SONARQUBE_PROJECT_KEY_ANNOTATION + +Exports that been introduced to plugin-sonarqube-react are documented in the [API report](https://github.com/backstage/backstage/blob/master/plugins/sonarqube-react/api-report.md). From 5a18e01bfa18d9bc5f622faf99ec53d5e6bec36a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 14 Dec 2022 22:52:38 +0100 Subject: [PATCH 112/237] backend-test-utils: check afterEach -> afterAll Signed-off-by: Patrik Oldsberg --- packages/backend-test-utils/src/next/wiring/TestBackend.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.ts index 5558a74ca1..1d401bbf6f 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.ts @@ -129,7 +129,7 @@ export async function startTestBackend< let registered = false; function registerTestHooks() { - if (typeof afterEach !== 'function') { + if (typeof afterAll !== 'function') { return; } if (registered) { From f7ccaef866387918ec819555d3f878b14ac1ba7a Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Wed, 23 Nov 2022 17:48:19 -0500 Subject: [PATCH 113/237] convert fetchObjectsForService mocks to msw this will allow some dramatic refactoring Signed-off-by: Jamie Klassen --- .../src/service/KubernetesFetcher.test.ts | 462 ++++++++---------- 1 file changed, 197 insertions(+), 265 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts index c2560e58b0..5fd9c9bf83 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts @@ -16,8 +16,12 @@ import { getVoidLogger } from '@backstage/backend-common'; import { KubernetesClientBasedFetcher } from './KubernetesFetcher'; +import { KubernetesClientProvider } from './KubernetesClientProvider'; import { ObjectToFetch } from '../types/types'; import { topPods } from '@kubernetes/client-node'; +import { MockedRequest, rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; jest.mock('@kubernetes/client-node', () => ({ ...jest.requireActual('@kubernetes/client-node'), @@ -56,45 +60,43 @@ const POD_METRICS_FIXTURE = { describe('KubernetesFetcher', () => { describe('fetchObjectsForService', () => { - let clientMock: any; - let kubernetesClientProvider: any; let sut: KubernetesClientBasedFetcher; + const worker = setupServer(); + setupRequestMockHandlers(worker); - beforeEach(() => { - jest.resetAllMocks(); - clientMock = { - listClusterCustomObject: jest.fn(), - listNamespacedCustomObject: jest.fn(), - addInterceptor: jest.fn(), - }; - - kubernetesClientProvider = { - getCustomObjectsClient: jest.fn(() => clientMock), - }; - - sut = new KubernetesClientBasedFetcher({ - kubernetesClientProvider, - logger: getVoidLogger(), - }); - }); + const labels = (req: MockedRequest): object => { + const selectorParam = req.url.searchParams.get('labelSelector'); + if (selectorParam) { + const [key, value] = selectorParam.split('='); + return { [key]: value }; + } + return {}; + }; const testErrorResponse = async ( errorResponse: any, expectedResult: any, ) => { - clientMock.listClusterCustomObject.mockResolvedValueOnce({ - body: { - items: [ - { - metadata: { - name: 'pod-name', - }, - }, - ], - }, - }); - - clientMock.listClusterCustomObject.mockRejectedValue(errorResponse); + worker.use( + rest.get('http://localhost:9999/api/v1/pods', (req, res, ctx) => + res( + ctx.json({ + items: [{ metadata: { name: 'pod-name', labels: labels(req) } }], + }), + ), + ), + rest.get('http://localhost:9999/api/v1/services', (_, res, ctx) => { + return res( + ctx.status(errorResponse.response.statusCode), + ctx.json({ + kind: 'Status', + apiVersion: 'v1', + status: 'Failure', + code: errorResponse.response.statusCode, + }), + ); + }), + ); const result = await sut.fetchObjectsForService({ serviceId: 'some-service', @@ -118,66 +120,41 @@ describe('KubernetesFetcher', () => { { metadata: { name: 'pod-name', + labels: { 'backstage.io/kubernetes-id': 'some-service' }, }, }, ], }, ], }); - - expect(clientMock.listClusterCustomObject.mock.calls.length).toBe(2); - - expect(clientMock.listClusterCustomObject.mock.calls[0]).toEqual([ - '', - 'v1', - 'pods', - '', - false, - '', - '', - 'backstage.io/kubernetes-id=some-service', - ]); - - expect(clientMock.listClusterCustomObject.mock.calls[1]).toEqual([ - '', - 'v1', - 'services', - '', - false, - '', - '', - 'backstage.io/kubernetes-id=some-service', - ]); - - expect( - kubernetesClientProvider.getCustomObjectsClient.mock.calls.length, - ).toBe(2); }; - it('should return pods, services', async () => { - clientMock.listClusterCustomObject.mockResolvedValueOnce({ - body: { - items: [ - { - metadata: { - name: 'pod-name', - }, - }, - ], - }, + beforeEach(() => { + sut = new KubernetesClientBasedFetcher({ + kubernetesClientProvider: new KubernetesClientProvider(), + logger: getVoidLogger(), }); + }); - clientMock.listClusterCustomObject.mockResolvedValueOnce({ - body: { - items: [ - { - metadata: { - name: 'service-name', - }, - }, - ], - }, - }); + it('should return pods, services', async () => { + worker.use( + rest.get('http://localhost:9999/api/v1/pods', (req, res, ctx) => + res( + ctx.json({ + items: [{ metadata: { name: 'pod-name', labels: labels(req) } }], + }), + ), + ), + rest.get('http://localhost:9999/api/v1/services', (req, res, ctx) => + res( + ctx.json({ + items: [ + { metadata: { name: 'service-name', labels: labels(req) } }, + ], + }), + ), + ), + ); const result = await sut.fetchObjectsForService({ serviceId: 'some-service', @@ -201,6 +178,7 @@ describe('KubernetesFetcher', () => { { metadata: { name: 'pod-name', + labels: { 'backstage.io/kubernetes-id': 'some-service' }, }, }, ], @@ -211,77 +189,44 @@ describe('KubernetesFetcher', () => { { metadata: { name: 'service-name', + labels: { 'backstage.io/kubernetes-id': 'some-service' }, }, }, ], }, ], }); - - expect(clientMock.listClusterCustomObject.mock.calls.length).toBe(2); - - expect(clientMock.listClusterCustomObject.mock.calls[0]).toEqual([ - '', - 'v1', - 'pods', - '', - false, - '', - '', - 'backstage.io/kubernetes-id=some-service', - ]); - - expect(clientMock.listClusterCustomObject.mock.calls[1]).toEqual([ - '', - 'v1', - 'services', - '', - false, - '', - '', - 'backstage.io/kubernetes-id=some-service', - ]); - - expect( - kubernetesClientProvider.getCustomObjectsClient.mock.calls.length, - ).toBe(2); }); it('should return pods, services and customobjects', async () => { - clientMock.listClusterCustomObject.mockResolvedValueOnce({ - body: { - items: [ - { - metadata: { - name: 'pod-name', - }, - }, - ], - }, - }); - - clientMock.listClusterCustomObject.mockResolvedValueOnce({ - body: { - items: [ - { - metadata: { - name: 'service-name', - }, - }, - ], - }, - }); - - clientMock.listClusterCustomObject.mockResolvedValueOnce({ - body: { - items: [ - { - metadata: { - name: 'something-else', - }, - }, - ], - }, - }); + worker.use( + rest.get('http://localhost:9999/api/v1/pods', (req, res, ctx) => + res( + ctx.json({ + items: [{ metadata: { name: 'pod-name', labels: labels(req) } }], + }), + ), + ), + rest.get('http://localhost:9999/api/v1/services', (req, res, ctx) => + res( + ctx.json({ + items: [ + { metadata: { name: 'service-name', labels: labels(req) } }, + ], + }), + ), + ), + rest.get( + 'http://localhost:9999/apis/some-group/v2/things', + (req, res, ctx) => + res( + ctx.json({ + items: [ + { metadata: { name: 'something-else', labels: labels(req) } }, + ], + }), + ), + ), + ); const result = await sut.fetchObjectsForService({ serviceId: 'some-service', @@ -312,6 +257,7 @@ describe('KubernetesFetcher', () => { { metadata: { name: 'pod-name', + labels: { 'backstage.io/kubernetes-id': 'some-service' }, }, }, ], @@ -322,6 +268,7 @@ describe('KubernetesFetcher', () => { { metadata: { name: 'service-name', + labels: { 'backstage.io/kubernetes-id': 'some-service' }, }, }, ], @@ -332,51 +279,13 @@ describe('KubernetesFetcher', () => { { metadata: { name: 'something-else', + labels: { 'backstage.io/kubernetes-id': 'some-service' }, }, }, ], }, ], }); - - expect(clientMock.listClusterCustomObject.mock.calls.length).toBe(3); - - expect(clientMock.listClusterCustomObject.mock.calls[0]).toEqual([ - '', - 'v1', - 'pods', - '', - false, - '', - '', - 'backstage.io/kubernetes-id=some-service', - ]); - - expect(clientMock.listClusterCustomObject.mock.calls[1]).toEqual([ - '', - 'v1', - 'services', - '', - false, - '', - '', - 'backstage.io/kubernetes-id=some-service', - ]); - - expect(clientMock.listClusterCustomObject.mock.calls[2]).toEqual([ - 'some-group', - 'v2', - 'things', - '', - false, - '', - '', - 'backstage.io/kubernetes-id=some-service', - ]); - - expect( - kubernetesClientProvider.getCustomObjectsClient.mock.calls.length, - ).toBe(3); }); // they're in testErrorResponse // eslint-disable-next-line jest/expect-expect @@ -385,16 +294,11 @@ describe('KubernetesFetcher', () => { { response: { statusCode: 400, - request: { - uri: { - pathname: '/some/path', - }, - }, }, }, { errorType: 'BAD_REQUEST', - resourcePath: '/some/path', + resourcePath: '/api/v1/services', statusCode: 400, }, ); @@ -406,16 +310,11 @@ describe('KubernetesFetcher', () => { { response: { statusCode: 401, - request: { - uri: { - pathname: '/some/path', - }, - }, }, }, { errorType: 'UNAUTHORIZED_ERROR', - resourcePath: '/some/path', + resourcePath: '/api/v1/services', statusCode: 401, }, ); @@ -427,16 +326,11 @@ describe('KubernetesFetcher', () => { { response: { statusCode: 500, - request: { - uri: { - pathname: '/some/path', - }, - }, }, }, { errorType: 'SYSTEM_ERROR', - resourcePath: '/some/path', + resourcePath: '/api/v1/services', statusCode: 500, }, ); @@ -448,46 +342,36 @@ describe('KubernetesFetcher', () => { { response: { statusCode: 900, - request: { - uri: { - pathname: '/some/path', - }, - }, }, }, { errorType: 'UNKNOWN_ERROR', - resourcePath: '/some/path', + resourcePath: '/api/v1/services', statusCode: 900, }, ); }); - it('should always add a labelSelector query', async () => { - clientMock.listClusterCustomObject.mockResolvedValueOnce({ - body: { - items: [ - { - metadata: { - name: 'pod-name', - }, - }, - ], - }, - }); + it('should respect labelSelector', async () => { + worker.use( + rest.get('http://localhost:9999/api/v1/pods', (req, res, ctx) => + res( + ctx.json({ + items: [{ metadata: { name: 'pod-name', labels: labels(req) } }], + }), + ), + ), + rest.get('http://localhost:9999/api/v1/services', (req, res, ctx) => + res( + ctx.json({ + items: [ + { metadata: { name: 'service-name', labels: labels(req) } }, + ], + }), + ), + ), + ); - clientMock.listClusterCustomObject.mockResolvedValueOnce({ - body: { - items: [ - { - metadata: { - name: 'service-name', - }, - }, - ], - }, - }); - - await sut.fetchObjectsForService({ + const result = await sut.fetchObjectsForService({ serviceId: 'some-service', clusterDetails: { name: 'cluster1', @@ -496,41 +380,65 @@ describe('KubernetesFetcher', () => { authProvider: 'serviceAccount', }, objectTypesToFetch: OBJECTS_TO_FETCH, - labelSelector: '', + labelSelector: 'service-label=value', customResources: [], }); - const mockCall = clientMock.listClusterCustomObject.mock.calls[0]; - const actualSelector = mockCall[mockCall.length - 1]; - const expectedSelector = 'backstage.io/kubernetes-id=some-service'; - expect(actualSelector).toBe(expectedSelector); + expect(result).toStrictEqual({ + errors: [], + responses: [ + { + type: 'pods', + resources: [ + { + metadata: { + name: 'pod-name', + labels: { 'service-label': 'value' }, + }, + }, + ], + }, + { + type: 'services', + resources: [ + { + metadata: { + name: 'service-name', + labels: { 'service-label': 'value' }, + }, + }, + ], + }, + ], + }); }); it('should use namespace if provided', async () => { - clientMock.listNamespacedCustomObject.mockResolvedValueOnce({ - body: { - items: [ - { - metadata: { - name: 'pod-name', - }, - }, - ], - }, - }); + worker.use( + rest.get( + 'http://localhost:9999/api/v1/namespaces/some-namespace/pods', + (req, res, ctx) => + res( + ctx.json({ + items: [ + { metadata: { name: 'pod-name', labels: labels(req) } }, + ], + }), + ), + ), + rest.get( + 'http://localhost:9999/api/v1/namespaces/some-namespace/services', + (req, res, ctx) => + res( + ctx.json({ + items: [ + { metadata: { name: 'service-name', labels: labels(req) } }, + ], + }), + ), + ), + ); - clientMock.listNamespacedCustomObject.mockResolvedValueOnce({ - body: { - items: [ - { - metadata: { - name: 'service-name', - }, - }, - ], - }, - }); - - await sut.fetchObjectsForService({ + const result = await sut.fetchObjectsForService({ serviceId: 'some-service', clusterDetails: { name: 'cluster1', @@ -544,9 +452,33 @@ describe('KubernetesFetcher', () => { customResources: [], }); - const mockCall = clientMock.listNamespacedCustomObject.mock.calls[0]; - const namespace = mockCall[2]; - expect(namespace).toBe('some-namespace'); + expect(result).toStrictEqual({ + errors: [], + responses: [ + { + type: 'pods', + resources: [ + { + metadata: { + name: 'pod-name', + labels: { 'backstage.io/kubernetes-id': 'some-service' }, + }, + }, + ], + }, + { + type: 'services', + resources: [ + { + metadata: { + name: 'service-name', + labels: { 'backstage.io/kubernetes-id': 'some-service' }, + }, + }, + ], + }, + ], + }); }); }); From d31fe286250a82d56986740edda9d8baa7d8aeb8 Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Tue, 29 Nov 2022 07:35:50 -0500 Subject: [PATCH 114/237] backfill test for loading default kubeconfig Strictly speaking, the `loadFromDefault` method on `KubeConfig` as it is being called in `KubernetesClientProvider` has many other behaviours that this test does not exercise -- it has logic for reading kubeconfig files (based on an env var or default location in a home directory), provisions for running on Windows, and even contacting an apiserver on localhost. I'm making a breaking assumption here that these other scenarios are not important for Backstage users, and the one that really matters is the in-cluster one described in this test. Signed-off-by: Jamie Klassen --- .../src/service/KubernetesFetcher.test.ts | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts index 5fd9c9bf83..9984c66cc5 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts @@ -22,6 +22,7 @@ import { topPods } from '@kubernetes/client-node'; import { MockedRequest, rest } from 'msw'; import { setupServer } from 'msw/node'; import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import mockFs from 'mock-fs'; jest.mock('@kubernetes/client-node', () => ({ ...jest.requireActual('@kubernetes/client-node'), @@ -480,6 +481,73 @@ describe('KubernetesFetcher', () => { ], }); }); + describe('Backstage running on k8s', () => { + const initialHost = process.env.KUBERNETES_SERVICE_HOST; + const initialPort = process.env.KUBERNETES_SERVICE_PORT; + afterEach(() => { + process.env.KUBERNETES_SERVICE_HOST = initialHost; + process.env.KUBERNETES_SERVICE_PORT = initialPort; + mockFs.restore(); + }); + it('makes in-cluster requests when cluster details has no token', async () => { + process.env.KUBERNETES_SERVICE_HOST = '10.10.10.10'; + process.env.KUBERNETES_SERVICE_PORT = '443'; + mockFs({ + '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt': '', + '/var/run/secrets/kubernetes.io/serviceaccount/token': + 'allowed-token', + }); + worker.use( + rest.get('https://10.10.10.10/api/v1/pods', (req, res, ctx) => + req.headers.get('Authorization') === 'Bearer allowed-token' + ? res( + ctx.json({ + items: [ + { metadata: { name: 'pod-name', labels: labels(req) } }, + ], + }), + ) + : res(ctx.status(403)), + ), + ); + + const result = await sut.fetchObjectsForService({ + serviceId: 'some-service', + clusterDetails: { + name: 'overridden-to-in-cluster', + url: 'http://ignored', + authProvider: 'serviceAccount', + }, + objectTypesToFetch: new Set([ + { + group: '', + apiVersion: 'v1', + plural: 'pods', + objectType: 'pods', + }, + ]), + labelSelector: '', + customResources: [], + }); + + expect(result).toStrictEqual({ + errors: [], + responses: [ + { + type: 'pods', + resources: [ + { + metadata: { + name: 'pod-name', + labels: { 'backstage.io/kubernetes-id': 'some-service' }, + }, + }, + ], + }, + ], + }); + }); + }); }); describe('fetchPodMetricsByNamespaces', () => { From 15ad4b238952581e2612c35ee1ccb6095f06f28d Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Tue, 29 Nov 2022 09:06:59 -0500 Subject: [PATCH 115/237] stub k8s API checks bearer tokens Also, use idiomatic ResponseTransformers to clean up the individual tests. Finally, modify the "unauthorized error handling" spec to provide an invalid token to an otherwise-realistic stub k8s API. Signed-off-by: Jamie Klassen --- .../src/service/KubernetesFetcher.test.ts | 177 ++++++++++++------ 1 file changed, 123 insertions(+), 54 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts index 9984c66cc5..3bd38a4993 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts @@ -19,7 +19,13 @@ import { KubernetesClientBasedFetcher } from './KubernetesFetcher'; import { KubernetesClientProvider } from './KubernetesClientProvider'; import { ObjectToFetch } from '../types/types'; import { topPods } from '@kubernetes/client-node'; -import { MockedRequest, rest } from 'msw'; +import { + MockedRequest, + RestContext, + ResponseTransformer, + compose, + rest, +} from 'msw'; import { setupServer } from 'msw/node'; import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; import mockFs from 'mock-fs'; @@ -73,6 +79,37 @@ describe('KubernetesFetcher', () => { } return {}; }; + const checkToken = ( + req: MockedRequest, + ctx: RestContext, + token: string, + ): ResponseTransformer => { + switch (req.headers.get('Authorization')) { + case `Bearer ${token}`: + return ctx.status(200); + default: + return compose( + ctx.status(401), + ctx.json({ + kind: 'Status', + apiVersion: 'v1', + code: 401, + }), + ); + } + }; + const withLabels = ( + req: MockedRequest, + ctx: RestContext, + body: { items: { metadata: object }[] }, + ): ResponseTransformer => + ctx.json({ + ...body, + items: body.items.map(item => ({ + ...item, + metadata: { ...item.metadata, labels: labels(req) }, + })), + }); const testErrorResponse = async ( errorResponse: any, @@ -81,8 +118,9 @@ describe('KubernetesFetcher', () => { worker.use( rest.get('http://localhost:9999/api/v1/pods', (req, res, ctx) => res( - ctx.json({ - items: [{ metadata: { name: 'pod-name', labels: labels(req) } }], + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'pod-name' } }], }), ), ), @@ -141,17 +179,17 @@ describe('KubernetesFetcher', () => { worker.use( rest.get('http://localhost:9999/api/v1/pods', (req, res, ctx) => res( - ctx.json({ - items: [{ metadata: { name: 'pod-name', labels: labels(req) } }], + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'pod-name' } }], }), ), ), rest.get('http://localhost:9999/api/v1/services', (req, res, ctx) => res( - ctx.json({ - items: [ - { metadata: { name: 'service-name', labels: labels(req) } }, - ], + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'service-name' } }], }), ), ), @@ -202,17 +240,17 @@ describe('KubernetesFetcher', () => { worker.use( rest.get('http://localhost:9999/api/v1/pods', (req, res, ctx) => res( - ctx.json({ - items: [{ metadata: { name: 'pod-name', labels: labels(req) } }], + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'pod-name' } }], }), ), ), rest.get('http://localhost:9999/api/v1/services', (req, res, ctx) => res( - ctx.json({ - items: [ - { metadata: { name: 'service-name', labels: labels(req) } }, - ], + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'service-name' } }], }), ), ), @@ -220,10 +258,9 @@ describe('KubernetesFetcher', () => { 'http://localhost:9999/apis/some-group/v2/things', (req, res, ctx) => res( - ctx.json({ - items: [ - { metadata: { name: 'something-else', labels: labels(req) } }, - ], + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'something-else' } }], }), ), ), @@ -304,21 +341,56 @@ describe('KubernetesFetcher', () => { }, ); }); - // they're in testErrorResponse - // eslint-disable-next-line jest/expect-expect it('should return pods, unauthorized error', async () => { - await testErrorResponse( - { - response: { + worker.use( + rest.get('http://localhost:9999/api/v1/pods', (req, res, ctx) => + res( + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'pod-name' } }], + }), + ), + ), + rest.get('http://localhost:9999/api/v1/services', (req, res, ctx) => + res(checkToken(req, ctx, 'other-token')), + ), + ); + + const result = await sut.fetchObjectsForService({ + serviceId: 'some-service', + clusterDetails: { + name: 'cluster1', + url: 'http://localhost:9999', + serviceAccountToken: 'token', + authProvider: 'serviceAccount', + }, + objectTypesToFetch: OBJECTS_TO_FETCH, + labelSelector: '', + customResources: [], + }); + + expect(result).toStrictEqual({ + errors: [ + { + errorType: 'UNAUTHORIZED_ERROR', + resourcePath: '/api/v1/services', statusCode: 401, }, - }, - { - errorType: 'UNAUTHORIZED_ERROR', - resourcePath: '/api/v1/services', - statusCode: 401, - }, - ); + ], + responses: [ + { + type: 'pods', + resources: [ + { + metadata: { + name: 'pod-name', + labels: { 'backstage.io/kubernetes-id': 'some-service' }, + }, + }, + ], + }, + ], + }); }); // they're in testErrorResponse // eslint-disable-next-line jest/expect-expect @@ -356,17 +428,17 @@ describe('KubernetesFetcher', () => { worker.use( rest.get('http://localhost:9999/api/v1/pods', (req, res, ctx) => res( - ctx.json({ - items: [{ metadata: { name: 'pod-name', labels: labels(req) } }], + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'pod-name' } }], }), ), ), rest.get('http://localhost:9999/api/v1/services', (req, res, ctx) => res( - ctx.json({ - items: [ - { metadata: { name: 'service-name', labels: labels(req) } }, - ], + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'service-name' } }], }), ), ), @@ -419,10 +491,9 @@ describe('KubernetesFetcher', () => { 'http://localhost:9999/api/v1/namespaces/some-namespace/pods', (req, res, ctx) => res( - ctx.json({ - items: [ - { metadata: { name: 'pod-name', labels: labels(req) } }, - ], + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'pod-name' } }], }), ), ), @@ -430,10 +501,9 @@ describe('KubernetesFetcher', () => { 'http://localhost:9999/api/v1/namespaces/some-namespace/services', (req, res, ctx) => res( - ctx.json({ - items: [ - { metadata: { name: 'service-name', labels: labels(req) } }, - ], + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'service-name' } }], }), ), ), @@ -499,15 +569,14 @@ describe('KubernetesFetcher', () => { }); worker.use( rest.get('https://10.10.10.10/api/v1/pods', (req, res, ctx) => - req.headers.get('Authorization') === 'Bearer allowed-token' - ? res( - ctx.json({ - items: [ - { metadata: { name: 'pod-name', labels: labels(req) } }, - ], - }), - ) - : res(ctx.status(403)), + res( + checkToken(req, ctx, 'allowed-token'), + withLabels(req, ctx, { + items: [ + { metadata: { name: 'pod-name', labels: labels(req) } }, + ], + }), + ), ), ); From 030f9cb5c1caa2d30cb5341c64bb4b3183b5227a Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Tue, 29 Nov 2022 09:29:47 -0500 Subject: [PATCH 116/237] backfill test for non-kubernetes error Signed-off-by: Jamie Klassen --- .../src/service/KubernetesFetcher.test.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts index 3bd38a4993..8449a1716d 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts @@ -424,6 +424,40 @@ describe('KubernetesFetcher', () => { }, ); }); + it('fails on a network error', async () => { + worker.use( + rest.get('http://badurl.does.not.exist/api/v1/pods', (_, res) => + res.networkError('getaddrinfo ENOTFOUND badurl.does.not.exist'), + ), + rest.get( + 'http://badurl.does.not.exist/api/v1/services', + (req, res, ctx) => + res( + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'service-name' } }], + }), + ), + ), + ); + + const result = sut.fetchObjectsForService({ + serviceId: 'some-service', + clusterDetails: { + name: 'cluster1', + url: 'http://badurl.does.not.exist', + serviceAccountToken: 'token', + authProvider: 'serviceAccount', + }, + objectTypesToFetch: OBJECTS_TO_FETCH, + labelSelector: '', + customResources: [], + }); + + await expect(result).rejects.toThrow( + 'getaddrinfo ENOTFOUND badurl.does.not.exist', + ); + }); it('should respect labelSelector', async () => { worker.use( rest.get('http://localhost:9999/api/v1/pods', (req, res, ctx) => From 7e2cd3c3579606f93f90cbeec251c1a26ee1dc6d Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Tue, 29 Nov 2022 13:20:38 -0500 Subject: [PATCH 117/237] backfill test for warning log Signed-off-by: Jamie Klassen --- .../src/service/KubernetesFetcher.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts index 8449a1716d..0fc832e7bd 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts @@ -68,6 +68,7 @@ const POD_METRICS_FIXTURE = { describe('KubernetesFetcher', () => { describe('fetchObjectsForService', () => { let sut: KubernetesClientBasedFetcher; + const logger = getVoidLogger(); const worker = setupServer(); setupRequestMockHandlers(worker); @@ -171,7 +172,7 @@ describe('KubernetesFetcher', () => { beforeEach(() => { sut = new KubernetesClientBasedFetcher({ kubernetesClientProvider: new KubernetesClientProvider(), - logger: getVoidLogger(), + logger, }); }); @@ -341,7 +342,8 @@ describe('KubernetesFetcher', () => { }, ); }); - it('should return pods, unauthorized error', async () => { + it('should return pods and unauthorized error, logging a warning', async () => { + const warn = jest.spyOn(logger, 'warn'); worker.use( rest.get('http://localhost:9999/api/v1/pods', (req, res, ctx) => res( @@ -391,6 +393,9 @@ describe('KubernetesFetcher', () => { }, ], }); + expect(warn).toHaveBeenCalledWith( + 'statusCode=401 for resource /api/v1/services body=[{"kind":"Status","apiVersion":"v1","code":401}]', + ); }); // they're in testErrorResponse // eslint-disable-next-line jest/expect-expect From df716f5ed5a766636a9f257860b54ca9de146944 Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Tue, 29 Nov 2022 15:30:30 -0500 Subject: [PATCH 118/237] metrics tests use msw instead of mocks Signed-off-by: Jamie Klassen --- .../src/service/KubernetesFetcher.test.ts | 303 +++++++++++------- 1 file changed, 189 insertions(+), 114 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts index 0fc832e7bd..db318b4d0c 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts @@ -18,7 +18,6 @@ import { getVoidLogger } from '@backstage/backend-common'; import { KubernetesClientBasedFetcher } from './KubernetesFetcher'; import { KubernetesClientProvider } from './KubernetesClientProvider'; import { ObjectToFetch } from '../types/types'; -import { topPods } from '@kubernetes/client-node'; import { MockedRequest, RestContext, @@ -30,11 +29,6 @@ import { setupServer } from 'msw/node'; import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; import mockFs from 'mock-fs'; -jest.mock('@kubernetes/client-node', () => ({ - ...jest.requireActual('@kubernetes/client-node'), - topPods: jest.fn(), -})); - const OBJECTS_TO_FETCH = new Set([ { group: '', @@ -50,67 +44,69 @@ const OBJECTS_TO_FETCH = new Set([ }, ]); -const POD_METRICS_FIXTURE = { - containers: [], - cpu: { - currentUsage: 100, - limitTotal: 102, - requestTotal: 101, +const POD_METRICS_FIXTURE = [ + { + type: 'podstatus', + resources: [ + { + CPU: { CurrentUsage: 0, LimitTotal: 1, RequestTotal: 0.5 }, + Memory: { + CurrentUsage: 0, + LimitTotal: 1000000000n, + RequestTotal: 512000000n, + }, + }, + ], }, - memory: { - currentUsage: '1000', - limitTotal: '1002', - requestTotal: '1001', - }, - pod: {}, -}; +]; describe('KubernetesFetcher', () => { + const worker = setupServer(); + setupRequestMockHandlers(worker); + + const labels = (req: MockedRequest): object => { + const selectorParam = req.url.searchParams.get('labelSelector'); + if (selectorParam) { + const [key, value] = selectorParam.split('='); + return { [key]: value }; + } + return {}; + }; + const checkToken = ( + req: MockedRequest, + ctx: RestContext, + token: string, + ): ResponseTransformer => { + switch (req.headers.get('Authorization')) { + case `Bearer ${token}`: + return ctx.status(200); + default: + return compose( + ctx.status(401), + ctx.json({ + kind: 'Status', + apiVersion: 'v1', + code: 401, + }), + ); + } + }; + const withLabels = ( + req: MockedRequest, + ctx: RestContext, + body: T, + ): ResponseTransformer => + ctx.json({ + ...body, + items: body.items.map(item => ({ + ...item, + metadata: { ...item.metadata, labels: labels(req) }, + })), + }); + describe('fetchObjectsForService', () => { let sut: KubernetesClientBasedFetcher; const logger = getVoidLogger(); - const worker = setupServer(); - setupRequestMockHandlers(worker); - - const labels = (req: MockedRequest): object => { - const selectorParam = req.url.searchParams.get('labelSelector'); - if (selectorParam) { - const [key, value] = selectorParam.split('='); - return { [key]: value }; - } - return {}; - }; - const checkToken = ( - req: MockedRequest, - ctx: RestContext, - token: string, - ): ResponseTransformer => { - switch (req.headers.get('Authorization')) { - case `Bearer ${token}`: - return ctx.status(200); - default: - return compose( - ctx.status(401), - ctx.json({ - kind: 'Status', - apiVersion: 'v1', - code: 401, - }), - ); - } - }; - const withLabels = ( - req: MockedRequest, - ctx: RestContext, - body: { items: { metadata: object }[] }, - ): ResponseTransformer => - ctx.json({ - ...body, - items: body.items.map(item => ({ - ...item, - metadata: { ...item.metadata, labels: labels(req) }, - })), - }); const testErrorResponse = async ( errorResponse: any, @@ -611,9 +607,7 @@ describe('KubernetesFetcher', () => { res( checkToken(req, ctx, 'allowed-token'), withLabels(req, ctx, { - items: [ - { metadata: { name: 'pod-name', labels: labels(req) } }, - ], + items: [{ metadata: { name: 'pod-name' } }], }), ), ), @@ -659,25 +653,63 @@ describe('KubernetesFetcher', () => { }); describe('fetchPodMetricsByNamespaces', () => { - let kubernetesClientProvider: any; let sut: KubernetesClientBasedFetcher; beforeEach(() => { - jest.resetAllMocks(); - - kubernetesClientProvider = { - getMetricsClient: jest.fn(), - getCoreClientByClusterDetails: jest.fn(), - }; - sut = new KubernetesClientBasedFetcher({ - kubernetesClientProvider, + kubernetesClientProvider: new KubernetesClientProvider(), logger: getVoidLogger(), }); }); it('should return pod metrics', async () => { - (topPods as jest.Mock).mockResolvedValue(POD_METRICS_FIXTURE); + worker.use( + rest.get( + 'http://localhost:9999/api/v1/namespaces/:namespace/pods', + (req, res, ctx) => + res( + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [ + { + metadata: { name: 'pod-name' }, + spec: { + containers: [ + { + name: 'container-name', + resources: { + requests: { cpu: '500m', memory: '512M' }, + limits: { cpu: '1000m', memory: '1G' }, + }, + }, + ], + }, + }, + ], + }), + ), + ), + rest.get( + 'http://localhost:9999/apis/metrics.k8s.io/v1beta1/namespaces/:namespace/pods', + (req, res, ctx) => + res( + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [ + { + metadata: { name: 'pod-name' }, + containers: [ + { + name: 'container-name', + usage: { cpu: '0', memory: '0' }, + }, + ], + }, + ], + }), + ), + ), + ); const result = await sut.fetchPodMetricsByNamespaces( { @@ -686,36 +718,85 @@ describe('KubernetesFetcher', () => { serviceAccountToken: 'token', authProvider: 'serviceAccount', }, - new Set(['ns-a', 'ns-b']), + new Set(['ns-a']), ); - expect(result).toStrictEqual({ + expect(result).toMatchObject({ errors: [], - responses: [ - { - type: 'podstatus', - resources: POD_METRICS_FIXTURE, - }, - { - type: 'podstatus', - resources: POD_METRICS_FIXTURE, - }, - ], + responses: POD_METRICS_FIXTURE, }); }); it('should return pod metrics and error', async () => { - const topPodsMock = topPods as jest.Mock; - topPodsMock - .mockResolvedValueOnce(POD_METRICS_FIXTURE) - .mockRejectedValueOnce({ - response: { - statusCode: 404, - request: { - uri: { - pathname: '/some/path', - }, - }, - }, - }); + worker.use( + rest.get( + 'http://localhost:9999/api/v1/namespaces/ns-a/pods', + (req, res, ctx) => + res( + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [ + { + metadata: { name: 'pod-name' }, + spec: { + containers: [ + { + name: 'container-name', + resources: { + requests: { cpu: '500m', memory: '512M' }, + limits: { cpu: '1000m', memory: '1G' }, + }, + }, + ], + }, + }, + ], + }), + ), + ), + rest.get( + 'http://localhost:9999/apis/metrics.k8s.io/v1beta1/namespaces/ns-a/pods', + (req, res, ctx) => + res( + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [ + { + metadata: { name: 'pod-name' }, + containers: [ + { + name: 'container-name', + usage: { cpu: '0', memory: '0' }, + }, + ], + }, + ], + }), + ), + ), + rest.get( + 'http://localhost:9999/api/v1/namespaces/ns-b/pods', + (_, res, ctx) => + res( + ctx.status(404), + ctx.json({ + kind: 'Status', + apiVersion: 'v1', + code: 404, + }), + ), + ), + rest.get( + 'http://localhost:9999/apis/metrics.k8s.io/v1beta1/namespaces/ns-b/pods', + (_, res, ctx) => + res( + ctx.status(404), + ctx.json({ + kind: 'Status', + apiVersion: 'v1', + code: 404, + }), + ), + ), + ); const result = await sut.fetchPodMetricsByNamespaces( { @@ -726,21 +807,15 @@ describe('KubernetesFetcher', () => { }, new Set(['ns-a', 'ns-b']), ); - expect(result).toStrictEqual({ - errors: [ - { - errorType: 'NOT_FOUND', - resourcePath: '/some/path', - statusCode: 404, - }, - ], - responses: [ - { - type: 'podstatus', - resources: POD_METRICS_FIXTURE, - }, - ], - }); + + expect(result.errors).toStrictEqual([ + { + errorType: 'NOT_FOUND', + resourcePath: '/apis/metrics.k8s.io/v1beta1/namespaces/ns-b/pods', + statusCode: 404, + }, + ]); + expect(result.responses).toMatchObject(POD_METRICS_FIXTURE); }); }); }); From 99ca12978dd6ddd20f4d24ebbf259645bbb2f3ce Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Mon, 12 Dec 2022 18:02:33 -0500 Subject: [PATCH 119/237] spy on https calls to check caData This is a little awkward, but at the moment msw doesn't give much of a means of interacting with the non-WHATWG-Fetch-spec aspects of a request. Signed-off-by: Jamie Klassen --- .../src/service/KubernetesFetcher.test.ts | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts index db318b4d0c..b5ff2f35c3 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts @@ -520,6 +520,176 @@ describe('KubernetesFetcher', () => { ], }); }); + describe('when server uses TLS', () => { + let httpsRequest: jest.SpyInstance; + beforeAll(() => { + httpsRequest = jest.spyOn( + // this is pretty egregious reverse engineering of msw. + // If the SetupServerApi constructor was exported, we wouldn't need + // to be quite so hacky here + (worker as any).interceptor.interceptors[0].modules.get('https'), + 'request', + ); + }); + beforeEach(() => { + httpsRequest.mockClear(); + }); + it('should trust specified caData', async () => { + worker.use( + rest.get('https://localhost:9999/api/v1/pods', (req, res, ctx) => + res( + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'pod-name' } }], + }), + ), + ), + ); + + await sut.fetchObjectsForService({ + serviceId: 'some-service', + clusterDetails: { + name: 'cluster1', + url: 'https://localhost:9999', + serviceAccountToken: 'token', + authProvider: 'serviceAccount', + caData: 'MOCKCA', + }, + objectTypesToFetch: new Set([ + { + group: '', + apiVersion: 'v1', + plural: 'pods', + objectType: 'pods', + }, + ]), + labelSelector: '', + customResources: [], + }); + + expect(httpsRequest).toHaveBeenCalledTimes(1); + const [[{ agent }]] = httpsRequest.mock.calls; + expect(agent.options.ca.toString('base64')).toMatch('MOCKCA'); + }); + it('should use default chain of trust when caData is unspecified', async () => { + worker.use( + rest.get('https://localhost:9999/api/v1/pods', (req, res, ctx) => + res( + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'pod-name' } }], + }), + ), + ), + ); + + await sut.fetchObjectsForService({ + serviceId: 'some-service', + clusterDetails: { + name: 'cluster1', + url: 'https://localhost:9999', + serviceAccountToken: 'token', + authProvider: 'serviceAccount', + }, + objectTypesToFetch: new Set([ + { + group: '', + apiVersion: 'v1', + plural: 'pods', + objectType: 'pods', + }, + ]), + labelSelector: '', + customResources: [], + }); + + expect(httpsRequest).toHaveBeenCalledTimes(1); + const [[{ agent }]] = httpsRequest.mock.calls; + expect(agent.options.ca).toBeUndefined(); + }); + describe('with a CA file on disk', () => { + afterEach(() => { + mockFs.restore(); + }); + it('should trust contents of specified caFile', async () => { + mockFs({ + '/path/to/ca.crt': 'MOCKCA', + }); + worker.use( + rest.get('https://localhost:9999/api/v1/pods', (req, res, ctx) => + res( + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'pod-name' } }], + }), + ), + ), + ); + + await sut.fetchObjectsForService({ + serviceId: 'some-service', + clusterDetails: { + name: 'cluster1', + url: 'https://localhost:9999', + serviceAccountToken: 'token', + authProvider: 'serviceAccount', + caFile: '/path/to/ca.crt', + }, + objectTypesToFetch: new Set([ + { + group: '', + apiVersion: 'v1', + plural: 'pods', + objectType: 'pods', + }, + ]), + labelSelector: '', + customResources: [], + }); + + expect(httpsRequest).toHaveBeenCalledTimes(1); + const [[{ agent }]] = httpsRequest.mock.calls; + expect(agent.options.ca.toString()).toEqual('MOCKCA'); + }); + }); + it('should accept unauthorized certs when skipTLSVerify is set', async () => { + worker.use( + rest.get('https://localhost:9999/api/v1/pods', (req, res, ctx) => + res( + checkToken(req, ctx, 'token'), + withLabels(req, ctx, { + items: [{ metadata: { name: 'pod-name' } }], + }), + ), + ), + ); + + await sut.fetchObjectsForService({ + serviceId: 'some-service', + clusterDetails: { + name: 'cluster1', + url: 'https://localhost:9999', + serviceAccountToken: 'token', + authProvider: 'serviceAccount', + skipTLSVerify: true, + }, + objectTypesToFetch: new Set([ + { + group: '', + apiVersion: 'v1', + plural: 'pods', + objectType: 'pods', + }, + ]), + labelSelector: '', + customResources: [], + }); + + expect(httpsRequest).toHaveBeenCalledTimes(1); + const [[{ agent }]] = httpsRequest.mock.calls; + expect(agent.options.rejectUnauthorized).toBe(false); + }); + }); it('should use namespace if provided', async () => { worker.use( rest.get( From f54362ad5713358100e5947f8a26d1287b0e544e Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Tue, 29 Nov 2022 13:52:09 -0500 Subject: [PATCH 120/237] use node-fetch for k8s resources The main advantage this has over using the list*CustomObject methods is that it won't reject non-OK statuses, so we won't have to be so careful about catching and rethrowing. Personally I find it a bit clearer to see the actual requests rather than the client methods with all those empty arguments, and a side benefit is that this implementation uses the fetch API -- under the covers, @kubernetes/client-node is using the deprecated request library. Signed-off-by: Jamie Klassen --- .../src/service/KubernetesFetcher.ts | 119 ++++++++++++------ 1 file changed, 80 insertions(+), 39 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts index fd6400529b..db2090a155 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts @@ -14,7 +14,13 @@ * limitations under the License. */ -import { topPods } from '@kubernetes/client-node'; +import { + Cluster, + KubeConfig, + User, + bufferFromFileOrString, + topPods, +} from '@kubernetes/client-node'; import lodash, { Dictionary } from 'lodash'; import { Logger } from 'winston'; import { @@ -32,6 +38,9 @@ import { PodStatusFetchResponse, } from '@backstage/plugin-kubernetes-common'; import { KubernetesClientProvider } from './KubernetesClientProvider'; +import fetch, { Headers, RequestInit } from 'node-fetch'; +import * as https from 'https'; +import fs from 'fs-extra'; export interface KubernetesClientBasedFetcherOptions { kubernetesClientProvider: KubernetesClientProvider; @@ -149,50 +158,82 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { labelSelector: string, objectType: KubernetesObjectTypes, namespace?: string, - ): Promise { - const customObjects = - this.kubernetesClientProvider.getCustomObjectsClient(clusterDetails); - - customObjects.addInterceptor((requestOptions: any) => { - requestOptions.uri = requestOptions.uri.replace('/apis//v1/', '/api/v1/'); - }); - + ): Promise { + const { group, apiVersion, plural } = resource; + const encode = (s: string) => encodeURIComponent(s); + let resourcePath = group + ? `/apis/${encode(group)}/${encode(apiVersion)}` + : `/api/${encode(apiVersion)}`; if (namespace) { - return customObjects - .listNamespacedCustomObject( - resource.group, - resource.apiVersion, - namespace, - resource.plural, - '', - false, - '', - '', - labelSelector, - ) - .then(r => { + resourcePath += `/namespaces/${encode(namespace)}`; + } + resourcePath += `/${encode(plural)}`; + + const headers: Headers = new Headers({ + Accept: 'application/json', + 'Content-Type': 'application/json', + }); + const fetchOptions: RequestInit = { + method: 'GET', + }; + let token: Buffer | string; + let url: URL; + if (clusterDetails.serviceAccountToken) { + url = new URL(`${clusterDetails.url}${resourcePath}`); + + if (url.protocol === 'https:') { + fetchOptions.agent = new https.Agent({ + ca: + bufferFromFileOrString( + clusterDetails.caFile, + clusterDetails.caData, + ) ?? undefined, + rejectUnauthorized: !clusterDetails.skipTLSVerify, + }); + } + + token = clusterDetails.serviceAccountToken; + } else { + const kc = new KubeConfig(); + kc.loadFromCluster(); + // loadFromCluster never fails (unless an exception is thrown) and is + // guaranteed to populate the cluster/user/context correctly + const cluster = kc.getCurrentCluster() as Cluster; + const user = kc.getCurrentUser() as User; + url = new URL(`${cluster.server}${resourcePath}`); + + if (url.protocol === 'https:') { + fetchOptions.agent = new https.Agent({ + ca: fs.readFileSync(cluster.caFile as string), + }); + } + + token = fs.readFileSync(user.authProvider.config.tokenFile); + } + + headers.set('Authorization', `Bearer ${token}`); + fetchOptions.headers = headers; + url.search = `labelSelector=${labelSelector}`; + + return fetch(url.toString(), fetchOptions).then(r => { + return r.json().then(j => { + if (r.ok) { return { type: objectType, - resources: (r.body as any).items, + resources: j.items, }; - }); - } - return customObjects - .listClusterCustomObject( - resource.group, - resource.apiVersion, - resource.plural, - '', - false, - '', - '', - labelSelector, - ) - .then(r => { + } + this.logger.warn( + `statusCode=${ + r.status + } for resource ${resourcePath} body=[${JSON.stringify(j)}]`, + ); return { - type: objectType, - resources: (r.body as any).items, + errorType: statusCodeToErrorType(r.status), + statusCode: r.status, + resourcePath, }; }); + }); } } From fcf8d33014886ed2e0f3dc379f314a6e9e8bcb94 Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Wed, 30 Nov 2022 10:16:12 -0500 Subject: [PATCH 121/237] node-fetch-based implementation for pod metrics which eliminates the need for the KubernetesClientProvider Signed-off-by: Jamie Klassen --- plugins/kubernetes-backend/api-report.md | 16 -- .../src/service/KubernetesBuilder.ts | 2 - .../service/KubernetesClientProvider.test.ts | 95 ------- .../src/service/KubernetesClientProvider.ts | 87 ------- .../src/service/KubernetesFetcher.test.ts | 3 - .../src/service/KubernetesFetcher.ts | 243 ++++++++++-------- .../kubernetes-backend/src/service/index.ts | 1 - 7 files changed, 129 insertions(+), 318 deletions(-) delete mode 100644 plugins/kubernetes-backend/src/service/KubernetesClientProvider.test.ts delete mode 100644 plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts diff --git a/plugins/kubernetes-backend/api-report.md b/plugins/kubernetes-backend/api-report.md index 2161df7c9a..4ec043a8f9 100644 --- a/plugins/kubernetes-backend/api-report.md +++ b/plugins/kubernetes-backend/api-report.md @@ -5,21 +5,17 @@ ```ts import { CatalogApi } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; -import { CoreV1Api } from '@kubernetes/client-node'; import { Credentials } from 'aws-sdk'; -import { CustomObjectsApi } from '@kubernetes/client-node'; import type { CustomResourceMatcher } from '@backstage/plugin-kubernetes-common'; import { Duration } from 'luxon'; import { Entity } from '@backstage/catalog-model'; import express from 'express'; import type { FetchResponse } from '@backstage/plugin-kubernetes-common'; import type { JsonObject } from '@backstage/types'; -import { KubeConfig } from '@kubernetes/client-node'; import type { KubernetesFetchError } from '@backstage/plugin-kubernetes-common'; import { KubernetesRequestAuth } from '@backstage/plugin-kubernetes-common'; import type { KubernetesRequestBody } from '@backstage/plugin-kubernetes-common'; import { Logger } from 'winston'; -import { Metrics } from '@kubernetes/client-node'; import type { ObjectsByEntityResponse } from '@backstage/plugin-kubernetes-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import type { RequestHandler } from 'express'; @@ -262,18 +258,6 @@ export type KubernetesBuilderReturn = Promise<{ serviceLocator: KubernetesServiceLocator; }>; -// @alpha (undocumented) -export class KubernetesClientProvider { - // (undocumented) - getCoreClientByClusterDetails(clusterDetails: ClusterDetails): CoreV1Api; - // (undocumented) - getCustomObjectsClient(clusterDetails: ClusterDetails): CustomObjectsApi; - // (undocumented) - getKubeConfig(clusterDetails: ClusterDetails): KubeConfig; - // (undocumented) - getMetricsClient(clusterDetails: ClusterDetails): Metrics; -} - // @alpha export interface KubernetesClustersSupplier { getClusters(): Promise; diff --git a/plugins/kubernetes-backend/src/service/KubernetesBuilder.ts b/plugins/kubernetes-backend/src/service/KubernetesBuilder.ts index c8ff98f592..1e7b704ec5 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesBuilder.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesBuilder.ts @@ -34,7 +34,6 @@ import { ObjectsByEntityRequest, ServiceLocatorMethod, } from '../types/types'; -import { KubernetesClientProvider } from './KubernetesClientProvider'; import { DEFAULT_OBJECTS, KubernetesFanOutHandler, @@ -211,7 +210,6 @@ export class KubernetesBuilder { protected buildFetcher(): KubernetesFetcher { this.fetcher = new KubernetesClientBasedFetcher({ - kubernetesClientProvider: new KubernetesClientProvider(), logger: this.env.logger, }); diff --git a/plugins/kubernetes-backend/src/service/KubernetesClientProvider.test.ts b/plugins/kubernetes-backend/src/service/KubernetesClientProvider.test.ts deleted file mode 100644 index 0f73d129b3..0000000000 --- a/plugins/kubernetes-backend/src/service/KubernetesClientProvider.test.ts +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import '@backstage/backend-common'; -import { KubernetesClientProvider } from './KubernetesClientProvider'; -import { ClusterDetails } from '../types/types'; -import * as https from 'https'; -import mockFs from 'mock-fs'; - -describe('KubernetesClientProvider', () => { - beforeEach(() => { - jest.resetAllMocks(); - }); - afterEach(() => { - mockFs.restore(); - }); - - it('can get core client by cluster details', () => { - const sut = new KubernetesClientProvider(); - const getKubeConfig = jest.spyOn(sut, 'getKubeConfig'); - - const clusterDetails: ClusterDetails = { - name: 'cluster-name', - url: 'http://localhost:9999', - serviceAccountToken: 'TOKEN', - authProvider: 'serviceAccount', - }; - const result = sut.getCoreClientByClusterDetails(clusterDetails); - - expect(result.basePath).toBe('http://localhost:9999'); - // These fields aren't on the type but are there - const auth = (result as any).authentications.default; - expect(auth.users[0].token).toBe('TOKEN'); - expect(auth.clusters[0].name).toBe('cluster-name'); - expect(auth.clusters[0].skipTLSVerify).toBe(false); - - expect(getKubeConfig).toHaveBeenCalledTimes(1); - }); - - it('can get custom objects client by cluster details', () => { - const sut = new KubernetesClientProvider(); - const getKubeConfig = jest.spyOn(sut, 'getKubeConfig'); - - const clusterDetails: ClusterDetails = { - name: 'cluster-name', - url: 'http://localhost:9999', - serviceAccountToken: 'TOKEN', - authProvider: 'serviceAccount', - skipTLSVerify: false, - }; - const result = sut.getCustomObjectsClient(clusterDetails); - - expect(result.basePath).toBe('http://localhost:9999'); - // These fields aren't on the type but are there - const auth = (result as any).authentications.default; - expect(auth.users[0].token).toBe('TOKEN'); - expect(auth.clusters[0].name).toBe('cluster-name'); - - expect(getKubeConfig).toHaveBeenCalledTimes(1); - }); - - it('respects caFile', async () => { - mockFs({ - '/path/to/ca.crt': 'my-ca', - }); - const clusterDetails: ClusterDetails = { - name: 'cluster-name', - url: 'https://localhost:9999', - authProvider: 'serviceAccount', - serviceAccountToken: 'TOKEN', - caFile: '/path/to/ca.crt', - }; - const kubeConfig = new KubernetesClientProvider().getKubeConfig( - clusterDetails, - ); - - const options: https.RequestOptions = {}; - await kubeConfig.applytoHTTPSOptions(options); - - expect(options.ca?.toString()).toEqual('my-ca'); - }); -}); diff --git a/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts b/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts deleted file mode 100644 index 14c8e422fc..0000000000 --- a/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - Cluster, - Context, - CoreV1Api, - CustomObjectsApi, - KubeConfig, - Metrics, - User, -} from '@kubernetes/client-node'; -import { ClusterDetails } from '../types/types'; - -/** - * - * @alpha - */ -export class KubernetesClientProvider { - // visible for testing - getKubeConfig(clusterDetails: ClusterDetails): KubeConfig { - const cluster: Cluster = { - name: clusterDetails.name, - server: clusterDetails.url, - skipTLSVerify: clusterDetails.skipTLSVerify || false, - caData: clusterDetails.caData, - caFile: clusterDetails.caFile, - }; - - // TODO configure - const user: User = { - name: 'backstage', - token: clusterDetails.serviceAccountToken, - }; - - const context: Context = { - name: `${clusterDetails.name}`, - user: user.name, - cluster: cluster.name, - }; - - const kc: KubeConfig = new KubeConfig(); - if (clusterDetails.serviceAccountToken) { - kc.loadFromOptions({ - clusters: [cluster], - users: [user], - contexts: [context], - currentContext: context.name, - }); - } else { - kc.loadFromDefault(); - } - - return kc; - } - - getCoreClientByClusterDetails(clusterDetails: ClusterDetails): CoreV1Api { - const kc = this.getKubeConfig(clusterDetails); - - return kc.makeApiClient(CoreV1Api); - } - - getMetricsClient(clusterDetails: ClusterDetails): Metrics { - const kc = this.getKubeConfig(clusterDetails); - - return new Metrics(kc); - } - - getCustomObjectsClient(clusterDetails: ClusterDetails): CustomObjectsApi { - const kc = this.getKubeConfig(clusterDetails); - - return kc.makeApiClient(CustomObjectsApi); - } -} diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts index b5ff2f35c3..9a1833cc4c 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts @@ -16,7 +16,6 @@ import { getVoidLogger } from '@backstage/backend-common'; import { KubernetesClientBasedFetcher } from './KubernetesFetcher'; -import { KubernetesClientProvider } from './KubernetesClientProvider'; import { ObjectToFetch } from '../types/types'; import { MockedRequest, @@ -167,7 +166,6 @@ describe('KubernetesFetcher', () => { beforeEach(() => { sut = new KubernetesClientBasedFetcher({ - kubernetesClientProvider: new KubernetesClientProvider(), logger, }); }); @@ -827,7 +825,6 @@ describe('KubernetesFetcher', () => { beforeEach(() => { sut = new KubernetesClientBasedFetcher({ - kubernetesClientProvider: new KubernetesClientProvider(), logger: getVoidLogger(), }); }); diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts index db2090a155..f90b96644d 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts @@ -16,7 +16,9 @@ import { Cluster, + CoreV1Api, KubeConfig, + Metrics, User, bufferFromFileOrString, topPods, @@ -27,9 +29,7 @@ import { ClusterDetails, FetchResponseWrapper, KubernetesFetcher, - KubernetesObjectTypes, ObjectFetchParams, - ObjectToFetch, } from '../types/types'; import { FetchResponse, @@ -37,13 +37,11 @@ import { KubernetesErrorTypes, PodStatusFetchResponse, } from '@backstage/plugin-kubernetes-common'; -import { KubernetesClientProvider } from './KubernetesClientProvider'; -import fetch, { Headers, RequestInit } from 'node-fetch'; +import fetch, { RequestInit, Response } from 'node-fetch'; import * as https from 'https'; import fs from 'fs-extra'; export interface KubernetesClientBasedFetcherOptions { - kubernetesClientProvider: KubernetesClientProvider; logger: Logger; } @@ -81,14 +79,9 @@ const statusCodeToErrorType = (statusCode: number): KubernetesErrorTypes => { }; export class KubernetesClientBasedFetcher implements KubernetesFetcher { - private readonly kubernetesClientProvider: KubernetesClientProvider; private readonly logger: Logger; - constructor({ - kubernetesClientProvider, - logger, - }: KubernetesClientBasedFetcherOptions) { - this.kubernetesClientProvider = kubernetesClientProvider; + constructor({ logger }: KubernetesClientBasedFetcherOptions) { this.logger = logger; } @@ -97,16 +90,27 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { ): Promise { const fetchResults = Array.from(params.objectTypesToFetch) .concat(params.customResources) - .map(toFetch => { - return this.fetchResource( + .map(({ objectType, group, apiVersion, plural }) => + this.fetchResource( params.clusterDetails, - toFetch, + group, + apiVersion, + plural, + params.namespace, params.labelSelector || `backstage.io/kubernetes-id=${params.serviceId}`, - toFetch.objectType, - params.namespace, - ).catch(this.captureKubernetesErrorsRethrowOthers.bind(this)); - }); + ).then( + (r: Response): Promise => + r.ok + ? r.json().then( + ({ items }): FetchResponse => ({ + type: objectType, + resources: items, + }), + ) + : this.handleUnsuccessfulResponse(r), + ), + ); return Promise.all(fetchResults).then(fetchResultsToResponseWrapper); } @@ -115,51 +119,65 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { clusterDetails: ClusterDetails, namespaces: Set, ): Promise { - const metricsClient = - this.kubernetesClientProvider.getMetricsClient(clusterDetails); - const coreApi = - this.kubernetesClientProvider.getCoreClientByClusterDetails( - clusterDetails, - ); - - const fetchResults = Array.from(namespaces).map(ns => - topPods(coreApi, metricsClient, ns) - .then(r => { - return { + const fetchResults = Array.from(namespaces).map(async ns => { + const [podMetrics, podList] = await Promise.all([ + this.fetchResource( + clusterDetails, + 'metrics.k8s.io', + 'v1beta1', + 'pods', + ns, + ), + this.fetchResource(clusterDetails, '', 'v1', 'pods', ns), + ]); + if (podMetrics.ok && podList.ok) { + return topPods( + { + listPodForAllNamespaces: () => + podList.json().then(b => ({ body: b })), + } as unknown as CoreV1Api, + { + getPodMetrics: () => podMetrics.json(), + } as unknown as Metrics, + ).then( + (resources): PodStatusFetchResponse => ({ type: 'podstatus', - resources: r, - } as PodStatusFetchResponse; - }) - .catch(this.captureKubernetesErrorsRethrowOthers.bind(this)), - ); + resources, + }), + ); + } else if (podMetrics.ok) { + return this.handleUnsuccessfulResponse(podList); + } + return this.handleUnsuccessfulResponse(podMetrics); + }); return Promise.all(fetchResults).then(fetchResultsToResponseWrapper); } - private captureKubernetesErrorsRethrowOthers(e: any): KubernetesFetchError { - if (e.response && e.response.statusCode) { - this.logger.warn( - `statusCode=${e.response.statusCode} for resource ${ - e.response.request.uri.pathname - } body=[${JSON.stringify(e.response.body)}]`, - ); - return { - errorType: statusCodeToErrorType(e.response.statusCode), - statusCode: e.response.statusCode, - resourcePath: e.response.request.uri.pathname, - }; - } - throw e; + private async handleUnsuccessfulResponse( + res: Response, + ): Promise { + const resourcePath = new URL(res.url).pathname; + this.logger.warn( + `statusCode=${ + res.status + } for resource ${resourcePath} body=[${await res.text()}]`, + ); + return { + errorType: statusCodeToErrorType(res.status), + statusCode: res.status, + resourcePath, + }; } private fetchResource( clusterDetails: ClusterDetails, - resource: ObjectToFetch, - labelSelector: string, - objectType: KubernetesObjectTypes, + group: string, + apiVersion: string, + plural: string, namespace?: string, - ): Promise { - const { group, apiVersion, plural } = resource; + labelSelector?: string, + ): Promise { const encode = (s: string) => encodeURIComponent(s); let resourcePath = group ? `/apis/${encode(group)}/${encode(apiVersion)}` @@ -169,71 +187,68 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { } resourcePath += `/${encode(plural)}`; - const headers: Headers = new Headers({ - Accept: 'application/json', - 'Content-Type': 'application/json', - }); - const fetchOptions: RequestInit = { - method: 'GET', - }; - let token: Buffer | string; - let url: URL; - if (clusterDetails.serviceAccountToken) { - url = new URL(`${clusterDetails.url}${resourcePath}`); + const [url, requestInit]: [URL, RequestInit] = + clusterDetails.serviceAccountToken + ? this.fetchArgsFromClusterDetails(clusterDetails) + : this.fetchArgsInCluster(); - if (url.protocol === 'https:') { - fetchOptions.agent = new https.Agent({ - ca: - bufferFromFileOrString( - clusterDetails.caFile, - clusterDetails.caData, - ) ?? undefined, - rejectUnauthorized: !clusterDetails.skipTLSVerify, - }); - } - - token = clusterDetails.serviceAccountToken; - } else { - const kc = new KubeConfig(); - kc.loadFromCluster(); - // loadFromCluster never fails (unless an exception is thrown) and is - // guaranteed to populate the cluster/user/context correctly - const cluster = kc.getCurrentCluster() as Cluster; - const user = kc.getCurrentUser() as User; - url = new URL(`${cluster.server}${resourcePath}`); - - if (url.protocol === 'https:') { - fetchOptions.agent = new https.Agent({ - ca: fs.readFileSync(cluster.caFile as string), - }); - } - - token = fs.readFileSync(user.authProvider.config.tokenFile); + url.pathname = resourcePath; + if (labelSelector) { + url.search = `labelSelector=${labelSelector}`; } - headers.set('Authorization', `Bearer ${token}`); - fetchOptions.headers = headers; - url.search = `labelSelector=${labelSelector}`; + return fetch(url, requestInit); + } - return fetch(url.toString(), fetchOptions).then(r => { - return r.json().then(j => { - if (r.ok) { - return { - type: objectType, - resources: j.items, - }; - } - this.logger.warn( - `statusCode=${ - r.status - } for resource ${resourcePath} body=[${JSON.stringify(j)}]`, - ); - return { - errorType: statusCodeToErrorType(r.status), - statusCode: r.status, - resourcePath, - }; + private fetchArgsFromClusterDetails( + clusterDetails: ClusterDetails, + ): [URL, RequestInit] { + const requestInit: RequestInit = { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + Authorization: `Bearer ${clusterDetails.serviceAccountToken}`, + }, + }; + + const url: URL = new URL(clusterDetails.url); + if (url.protocol === 'https:') { + requestInit.agent = new https.Agent({ + ca: + bufferFromFileOrString( + clusterDetails.caFile, + clusterDetails.caData, + ) ?? undefined, + rejectUnauthorized: !clusterDetails.skipTLSVerify, }); - }); + } + return [url, requestInit]; + } + private fetchArgsInCluster(): [URL, RequestInit] { + const kc = new KubeConfig(); + kc.loadFromCluster(); + // loadFromCluster is guaranteed to populate the cluster/user/context + const cluster = kc.getCurrentCluster() as Cluster; + const user = kc.getCurrentUser() as User; + + const token = fs.readFileSync(user.authProvider.config.tokenFile); + + const requestInit: RequestInit = { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}`, + }, + }; + + const url = new URL(cluster.server); + if (url.protocol === 'https:') { + requestInit.agent = new https.Agent({ + ca: fs.readFileSync(cluster.caFile as string), + }); + } + return [url, requestInit]; } } diff --git a/plugins/kubernetes-backend/src/service/index.ts b/plugins/kubernetes-backend/src/service/index.ts index 70ff530bee..620f788ac2 100644 --- a/plugins/kubernetes-backend/src/service/index.ts +++ b/plugins/kubernetes-backend/src/service/index.ts @@ -15,7 +15,6 @@ */ export * from './KubernetesBuilder'; -export * from './KubernetesClientProvider'; export { DEFAULT_OBJECTS } from './KubernetesFanOutHandler'; export { HEADER_KUBERNETES_CLUSTER, KubernetesProxy } from './KubernetesProxy'; export * from './router'; From 0ad476a720f520b267e446f2d64c7cafdd8b5462 Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Wed, 30 Nov 2022 13:17:40 -0500 Subject: [PATCH 122/237] gracefully surface FetchErrors Signed-off-by: Jamie Klassen --- .../service/KubernetesFanOutHandler.test.ts | 101 ++++++++++++++++++ .../src/service/KubernetesFanOutHandler.ts | 9 ++ plugins/kubernetes-common/api-report.md | 27 +++-- plugins/kubernetes-common/src/types.ts | 11 +- .../src/components/ErrorPanel/ErrorPanel.tsx | 3 +- 5 files changed, 141 insertions(+), 10 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts index 1af5371e53..fcb8eceb19 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts @@ -20,8 +20,14 @@ import { CustomResource, FetchResponseWrapper, ObjectFetchParams, + KubernetesServiceLocator, } from '../types/types'; import { KubernetesFanOutHandler } from './KubernetesFanOutHandler'; +import { KubernetesClientBasedFetcher } from './KubernetesFetcher'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { ObjectsByEntityResponse } from '@backstage/plugin-kubernetes-common'; const fetchObjectsForService = jest.fn(); const fetchPodMetricsByNamespaces = jest.fn(); @@ -751,6 +757,101 @@ describe('getKubernetesObjectsByEntity', () => { ], }); }); + describe('with a real fetcher', () => { + const worker = setupServer(); + setupRequestMockHandlers(worker); + it('fetch error short-circuits requests to a single cluster, recovering across the fleet', async () => { + const pods = [{ metadata: { name: 'pod-name' } }]; + const services = [{ metadata: { name: 'service-name' } }]; + worker.use( + rest.get('https://works/api/v1/pods', (_, res, ctx) => + res(ctx.json({ items: pods })), + ), + rest.get('https://works/api/v1/services', (_, res, ctx) => + res(ctx.json({ items: services })), + ), + rest.get('https://fails/api/v1/pods', (_, res) => + res.networkError('socket error'), + ), + rest.get('https://fails/api/v1/services', (_, res, ctx) => + res(ctx.json({ items: services })), + ), + ); + const fleet: jest.Mocked = { + getClustersByEntity: jest.fn().mockResolvedValue({ + clusters: [ + { + name: 'works', + url: 'https://works', + authProvider: 'serviceAccount', + serviceAccountToken: 'token', + skipMetricsLookup: true, + }, + { + name: 'fails', + url: 'https://fails', + authProvider: 'serviceAccount', + serviceAccountToken: 'token', + skipMetricsLookup: true, + }, + ], + }), + }; + const logger = getVoidLogger(); + const sut = new KubernetesFanOutHandler({ + logger, + fetcher: new KubernetesClientBasedFetcher({ logger }), + serviceLocator: fleet, + customResources: [], + objectTypesToFetch: [ + { + group: '', + apiVersion: 'v1', + plural: 'pods', + objectType: 'pods', + }, + { + group: '', + apiVersion: 'v1', + plural: 'services', + objectType: 'services', + }, + ], + }); + + const result = await sut.getKubernetesObjectsByEntity({ + entity, + auth: {}, + }); + + const expected: ObjectsByEntityResponse = { + items: [ + { + cluster: { name: 'works' }, + resources: [ + { type: 'pods', resources: pods }, + { type: 'services', resources: services }, + ], + podMetrics: [], + errors: [], + }, + { + cluster: { name: 'fails' }, + resources: [], + podMetrics: [], + errors: [ + { + errorType: 'FETCH_ERROR', + message: + 'request to https://fails/api/v1/pods?labelSelector=backstage.io/kubernetes-id=test-component failed, reason: socket error', + }, + ], + }, + ], + }; + expect(result).toStrictEqual(expected); + }); + }); }); describe('getCustomResourcesByEntity', () => { diff --git a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.ts b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.ts index 1fa9d23837..bc8881f95b 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.ts @@ -274,6 +274,15 @@ export class KubernetesFanOutHandler { namespace, }) .then(result => this.getMetricsForPods(clusterDetailsItem, result)) + .catch((e): responseWithMetrics => { + return [ + { + errors: [{ errorType: 'FETCH_ERROR', message: e.message }], + responses: [], + }, + [], + ]; + }) .then(r => this.toClusterObjects(clusterDetailsItem, r)); }), ).then(this.toObjectsByEntityResponse); diff --git a/plugins/kubernetes-common/api-report.md b/plugins/kubernetes-common/api-report.md index 8509fa97cc..fe8aa19966 100644 --- a/plugins/kubernetes-common/api-report.md +++ b/plugins/kubernetes-common/api-report.md @@ -184,14 +184,7 @@ export type KubernetesErrorTypes = | 'UNKNOWN_ERROR'; // @public (undocumented) -export interface KubernetesFetchError { - // (undocumented) - errorType: KubernetesErrorTypes; - // (undocumented) - resourcePath?: string; - // (undocumented) - statusCode?: number; -} +export type KubernetesFetchError = StatusError | RawFetchError; // @public (undocumented) export interface KubernetesRequestAuth { @@ -241,6 +234,14 @@ export interface PodStatusFetchResponse { type: 'podstatus'; } +// @public (undocumented) +export interface RawFetchError { + // (undocumented) + errorType: 'FETCH_ERROR'; + // (undocumented) + message: string; +} + // @public (undocumented) export interface ReplicaSetsFetchResponse { // (undocumented) @@ -265,6 +266,16 @@ export interface StatefulSetsFetchResponse { type: 'statefulsets'; } +// @public (undocumented) +export interface StatusError { + // (undocumented) + errorType: KubernetesErrorTypes; + // (undocumented) + resourcePath?: string; + // (undocumented) + statusCode?: number; +} + // @public (undocumented) export interface WorkloadsByEntityRequest { // (undocumented) diff --git a/plugins/kubernetes-common/src/types.ts b/plugins/kubernetes-common/src/types.ts index 3feddd7f4c..f723a46d16 100644 --- a/plugins/kubernetes-common/src/types.ts +++ b/plugins/kubernetes-common/src/types.ts @@ -223,12 +223,21 @@ export interface PodStatusFetchResponse { } /** @public */ -export interface KubernetesFetchError { +export type KubernetesFetchError = StatusError | RawFetchError; + +/** @public */ +export interface StatusError { errorType: KubernetesErrorTypes; statusCode?: number; resourcePath?: string; } +/** @public */ +export interface RawFetchError { + errorType: 'FETCH_ERROR'; + message: string; +} + /** @public */ export type KubernetesErrorTypes = | 'BAD_REQUEST' diff --git a/plugins/kubernetes/src/components/ErrorPanel/ErrorPanel.tsx b/plugins/kubernetes/src/components/ErrorPanel/ErrorPanel.tsx index fcdeeef393..1e55cd8d18 100644 --- a/plugins/kubernetes/src/components/ErrorPanel/ErrorPanel.tsx +++ b/plugins/kubernetes/src/components/ErrorPanel/ErrorPanel.tsx @@ -29,7 +29,8 @@ const clustersWithErrorsToErrorMessage = ( {c.errors.map((e, j) => { return ( - {`Error fetching Kubernetes resource: '${e.resourcePath}', error: ${e.errorType}, status code: ${e.statusCode}`} + {e.errorType !== 'FETCH_ERROR' && + `Error fetching Kubernetes resource: '${e.resourcePath}', error: ${e.errorType}, status code: ${e.statusCode}`} ); })} From a5a9d3318ae01d8e66ade8fbaa12271a3386cd92 Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Wed, 30 Nov 2022 13:54:28 -0500 Subject: [PATCH 123/237] don't surface non-FetchErrors It makes sense to gracefully show an error that was actually related to fetching data from Kubernetes, but if the error is happening for some other exotic reason, the default error handling logic in Backstage should take over and log/manage it appropriately. Signed-off-by: Jamie Klassen --- .../service/KubernetesFanOutHandler.test.ts | 21 +++++++++++++++++ .../src/service/KubernetesFanOutHandler.ts | 23 +++++++++++-------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts index fcb8eceb19..17a86eb30d 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts @@ -757,6 +757,27 @@ describe('getKubernetesObjectsByEntity', () => { ], }); }); + it('fails when fetcher rejects with a non-FetchError', async () => { + const nonFetchError = new Error('not a fetch error'); + getClustersByEntity.mockResolvedValue({ + clusters: [ + { + name: 'test-cluster', + authProvider: 'serviceAccount', + skipMetricsLookup: true, + }, + ], + }); + fetchObjectsForService.mockRejectedValue(nonFetchError); + + const sut = getKubernetesFanOutHandler([]); + + const result = sut.getKubernetesObjectsByEntity({ + entity, + auth: {}, + }); + await expect(result).rejects.toThrow(nonFetchError); + }); describe('with a real fetcher', () => { const worker = setupServer(); setupRequestMockHandlers(worker); diff --git a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.ts b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.ts index bc8881f95b..a738c69bcc 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.ts @@ -274,15 +274,20 @@ export class KubernetesFanOutHandler { namespace, }) .then(result => this.getMetricsForPods(clusterDetailsItem, result)) - .catch((e): responseWithMetrics => { - return [ - { - errors: [{ errorType: 'FETCH_ERROR', message: e.message }], - responses: [], - }, - [], - ]; - }) + .catch( + (e): Promise => + e.name === 'FetchError' + ? Promise.resolve([ + { + errors: [ + { errorType: 'FETCH_ERROR', message: e.message }, + ], + responses: [], + }, + [], + ]) + : Promise.reject(e), + ) .then(r => this.toClusterObjects(clusterDetailsItem, r)); }), ).then(this.toObjectsByEntityResponse); From bacb5470b6a20212ebed7a20a510e0afc622ff13 Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Wed, 30 Nov 2022 14:09:00 -0500 Subject: [PATCH 124/237] frontend displays new error types Signed-off-by: Jamie Klassen --- .../components/ErrorPanel/ErrorPanel.test.tsx | 84 +++++++++++++------ .../src/components/ErrorPanel/ErrorPanel.tsx | 5 +- 2 files changed, 63 insertions(+), 26 deletions(-) diff --git a/plugins/kubernetes/src/components/ErrorPanel/ErrorPanel.test.tsx b/plugins/kubernetes/src/components/ErrorPanel/ErrorPanel.test.tsx index 832d6eed91..0d75c09a11 100644 --- a/plugins/kubernetes/src/components/ErrorPanel/ErrorPanel.test.tsx +++ b/plugins/kubernetes/src/components/ErrorPanel/ErrorPanel.test.tsx @@ -20,36 +20,14 @@ import { wrapInTestApp } from '@backstage/test-utils'; import { ErrorPanel } from './ErrorPanel'; describe('ErrorPanel', () => { - it('render with error message', async () => { - const { getByText } = render( - wrapInTestApp( - , - ), - ); - - // title - expect( - getByText( - 'There was a problem retrieving some Kubernetes resources for the entity: THIS_ENTITY. This could mean that the Error Reporting card is not completely accurate.', - ), - ).toBeInTheDocument(); - - // message - expect(getByText('Errors: SOME_ERROR_MESSAGE')).toBeInTheDocument(); - }); - it('render with cluster errors', async () => { + it('displays path and status code when a cluster has an HTTP error', async () => { const { getByText } = render( wrapInTestApp( { ), ).toBeInTheDocument(); }); + it('displays message for non-HTTP-status-related fetch errors', async () => { + const { getByText } = render( + wrapInTestApp( + , + ), + ); + + // title + expect( + getByText( + 'There was a problem retrieving some Kubernetes resources for the entity: THIS_ENTITY. This could mean that the Error Reporting card is not completely accurate.', + ), + ).toBeInTheDocument(); + + // message + expect(getByText('Errors:')).toBeInTheDocument(); + expect(getByText('Cluster: THIS_CLUSTER')).toBeInTheDocument(); + expect( + getByText( + 'Error communicating with Kubernetes: FETCH_ERROR, message: description of error', + ), + ).toBeInTheDocument(); + }); + it('displays error message', async () => { + const { getByText } = render( + wrapInTestApp( + , + ), + ); + + // title + expect( + getByText( + 'There was a problem retrieving some Kubernetes resources for the entity: THIS_ENTITY. This could mean that the Error Reporting card is not completely accurate.', + ), + ).toBeInTheDocument(); + + // message + expect(getByText('Errors: SOME_ERROR_MESSAGE')).toBeInTheDocument(); + }); }); diff --git a/plugins/kubernetes/src/components/ErrorPanel/ErrorPanel.tsx b/plugins/kubernetes/src/components/ErrorPanel/ErrorPanel.tsx index 1e55cd8d18..d2cb88b9ca 100644 --- a/plugins/kubernetes/src/components/ErrorPanel/ErrorPanel.tsx +++ b/plugins/kubernetes/src/components/ErrorPanel/ErrorPanel.tsx @@ -29,8 +29,9 @@ const clustersWithErrorsToErrorMessage = ( {c.errors.map((e, j) => { return ( - {e.errorType !== 'FETCH_ERROR' && - `Error fetching Kubernetes resource: '${e.resourcePath}', error: ${e.errorType}, status code: ${e.statusCode}`} + {e.errorType === 'FETCH_ERROR' + ? `Error communicating with Kubernetes: ${e.errorType}, message: ${e.message}` + : `Error fetching Kubernetes resource: '${e.resourcePath}', error: ${e.errorType}, status code: ${e.statusCode}`} ); })} From b008ccee54fbb1848e49e117551bfdf339257967 Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Wed, 30 Nov 2022 22:56:06 -0500 Subject: [PATCH 125/237] throw when missing token and not on k8s Signed-off-by: Jamie Klassen --- .../src/service/KubernetesFetcher.test.ts | 18 ++++++++++++++++++ .../src/service/KubernetesFetcher.ts | 18 ++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts index 9a1833cc4c..2311dacdc3 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts @@ -754,6 +754,24 @@ describe('KubernetesFetcher', () => { ], }); }); + describe('Backstage not running on k8s', () => { + it('fails if cluster details has no token', () => { + const result = sut.fetchObjectsForService({ + serviceId: 'some-service', + clusterDetails: { + name: 'unauthenticated-cluster', + url: 'http://ignored', + authProvider: 'serviceAccount', + }, + objectTypesToFetch: OBJECTS_TO_FETCH, + labelSelector: '', + customResources: [], + }); + return expect(result).rejects.toThrow( + "no bearer token for cluster 'unauthenticated-cluster' and not running in Kubernetes", + ); + }); + }); describe('Backstage running on k8s', () => { const initialHost = process.env.KUBERNETES_SERVICE_HOST; const initialPort = process.env.KUBERNETES_SERVICE_PORT; diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts index f90b96644d..b700c74e4c 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts @@ -15,6 +15,7 @@ */ import { + Config, Cluster, CoreV1Api, KubeConfig, @@ -187,10 +188,19 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { } resourcePath += `/${encode(plural)}`; - const [url, requestInit]: [URL, RequestInit] = - clusterDetails.serviceAccountToken - ? this.fetchArgsFromClusterDetails(clusterDetails) - : this.fetchArgsInCluster(); + let url: URL; + let requestInit: RequestInit; + if (clusterDetails.serviceAccountToken) { + [url, requestInit] = this.fetchArgsFromClusterDetails(clusterDetails); + } else if (fs.pathExistsSync(Config.SERVICEACCOUNT_TOKEN_PATH)) { + [url, requestInit] = this.fetchArgsInCluster(); + } else { + return Promise.reject( + new Error( + `no bearer token for cluster '${clusterDetails.name}' and not running in Kubernetes`, + ), + ); + } url.pathname = resourcePath; if (labelSelector) { From 7341131af5c33b1d11378ebb5baa05071caf2eb5 Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Wed, 14 Dec 2022 17:49:56 -0500 Subject: [PATCH 126/237] warning is a full sentence mentioning cluster name This seems like a more helpful log message for operators, especially those with many clusters. Signed-off-by: Jamie Klassen --- .../src/service/KubernetesFetcher.test.ts | 2 +- .../src/service/KubernetesFetcher.ts | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts index 2311dacdc3..065416f8c1 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts @@ -388,7 +388,7 @@ describe('KubernetesFetcher', () => { ], }); expect(warn).toHaveBeenCalledWith( - 'statusCode=401 for resource /api/v1/services body=[{"kind":"Status","apiVersion":"v1","code":401}]', + 'Received 401 status when fetching "/api/v1/services" from cluster "cluster1"; body=[{"kind":"Status","apiVersion":"v1","code":401}]', ); }); // they're in testErrorResponse diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts index b700c74e4c..6ef4397e9c 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts @@ -109,7 +109,7 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { resources: items, }), ) - : this.handleUnsuccessfulResponse(r), + : this.handleUnsuccessfulResponse(params.clusterDetails.name, r), ), ); @@ -147,22 +147,23 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { }), ); } else if (podMetrics.ok) { - return this.handleUnsuccessfulResponse(podList); + return this.handleUnsuccessfulResponse(clusterDetails.name, podList); } - return this.handleUnsuccessfulResponse(podMetrics); + return this.handleUnsuccessfulResponse(clusterDetails.name, podMetrics); }); return Promise.all(fetchResults).then(fetchResultsToResponseWrapper); } private async handleUnsuccessfulResponse( + clusterName: string, res: Response, ): Promise { const resourcePath = new URL(res.url).pathname; this.logger.warn( - `statusCode=${ + `Received ${ res.status - } for resource ${resourcePath} body=[${await res.text()}]`, + } status when fetching "${resourcePath}" from cluster "${clusterName}"; body=[${await res.text()}]`, ); return { errorType: statusCodeToErrorType(res.status), From 2db8acffe7e1c12e633eede6eccd619ce22e621d Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Wed, 14 Dec 2022 17:55:53 -0500 Subject: [PATCH 127/237] add changeset Signed-off-by: Jamie Klassen --- .changeset/brave-eggs-impress.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/brave-eggs-impress.md diff --git a/.changeset/brave-eggs-impress.md b/.changeset/brave-eggs-impress.md new file mode 100644 index 0000000000..a0df24d8f9 --- /dev/null +++ b/.changeset/brave-eggs-impress.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-kubernetes-backend': minor +'@backstage/plugin-kubernetes-common': minor +'@backstage/plugin-kubernetes': patch +--- + +Kubernetes plugin now gracefully surfaces transport-level errors (like DNS or timeout, or other socket errors) occurring while fetching data. This will be merged into any data that is fetched successfully, fixing a bug where the whole page would be empty if any fetch operation encountered such an error. From bcd800bb33b8a18eb2e587adfabc7daa1e73d2b6 Mon Sep 17 00:00:00 2001 From: iris Date: Thu, 15 Dec 2022 15:28:56 +0800 Subject: [PATCH 128/237] make last_updated_at optional Signed-off-by: iris --- plugins/catalog-backend/src/database/tables.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index 9d89b17d16..e3f3efd1bb 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -66,7 +66,7 @@ export type DbFinalEntitiesRow = { hash: string; stitch_ticket: string; final_entity?: string; - last_updated_at: string | Date; + last_updated_at?: string | Date; }; export type DbSearchRow = { From 860fbcde3b9d154a19411d638e339c47be8086b7 Mon Sep 17 00:00:00 2001 From: iris Date: Thu, 15 Dec 2022 15:30:52 +0800 Subject: [PATCH 129/237] add ts-check Signed-off-by: iris --- .../20221201085245_add_last_updated_at_in_final_entities.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js index e34d45ade6..44fcf30ba4 100644 --- a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js +++ b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js @@ -14,6 +14,11 @@ * limitations under the License. */ +// @ts-check + +/** + * @param { import("knex").Knex } knex + */ exports.up = async function up(knex) { await knex.schema.table('final_entities', table => { table From b568378cf851ca29467685b797b73b15539c772c Mon Sep 17 00:00:00 2001 From: iris Date: Thu, 15 Dec 2022 15:44:09 +0800 Subject: [PATCH 130/237] add migration testing for 20221201085245_add_last_updated_at_in_final_entities Signed-off-by: iris --- .../catalog-backend/src/migrations.test.ts | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/plugins/catalog-backend/src/migrations.test.ts b/plugins/catalog-backend/src/migrations.test.ts index 9d9b02fa82..f1ed40dbbe 100644 --- a/plugins/catalog-backend/src/migrations.test.ts +++ b/plugins/catalog-backend/src/migrations.test.ts @@ -100,4 +100,82 @@ describe('migrations', () => { }, 60_000, ); + + it.each(databases.eachSupportedId())( + '20221201085245_add_last_updated_at_in_final_entities.js, %p', + async databaseId => { + const knex = await databases.init(databaseId); + + await migrateUntilBefore( + knex, + '20221201085245_add_last_updated_at_in_final_entities.js', + ); + + await knex + .insert([ + { + entity_id: 'my-id', + entity_ref: 'k:ns/n', + unprocessed_entity: JSON.stringify({}), + processed_entity: JSON.stringify({ + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + }, + spec: { + k: 'v', + }, + }), + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }, + ]) + .into('refresh_state'); + + await knex + .insert({ + entity_id: 'my-id', + hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e', + stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', + final_entity: + '{"apiVersion":"a","kind":"k","metadata":{"name":"n","namespace":"ns","uid":"my-id","etag":"d1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e"},"spec":{"k":"v"},"relations":[{"type":"looksAt","targetRef":"k:ns/other"}]}', + }) + .into('final_entities'); + + await migrateUpOnce(knex); + + await expect(knex('final_entities')).resolves.toEqual( + expect.arrayContaining([ + { + entity_id: 'my-id', + hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e', + stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', + final_entity: + '{"apiVersion":"a","kind":"k","metadata":{"name":"n","namespace":"ns","uid":"my-id","etag":"d1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e"},"spec":{"k":"v"},"relations":[{"type":"looksAt","targetRef":"k:ns/other"}]}', + last_updated_at: null, + }, + ]), + ); + + await migrateDownOnce(knex); + + await expect(knex('final_entities')).resolves.toEqual( + expect.arrayContaining([ + { + entity_id: 'my-id', + hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e', + stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', + final_entity: + '{"apiVersion":"a","kind":"k","metadata":{"name":"n","namespace":"ns","uid":"my-id","etag":"d1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e"},"spec":{"k":"v"},"relations":[{"type":"looksAt","targetRef":"k:ns/other"}]}', + }, + ]), + ); + + await knex.destroy(); + }, + 60_000, + ); }); From e5831e96d279ba6e9d67b946852b50d4755720fd Mon Sep 17 00:00:00 2001 From: Carlos Esteban Lopez Date: Thu, 15 Dec 2022 00:08:22 -0500 Subject: [PATCH 131/237] lint: Use rule excluding test files Signed-off-by: Carlos Esteban Lopez --- .eslintrc.js | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 4e6bbd3910..cf01ffe3a6 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -45,12 +45,6 @@ module.exports = { "CallExpression[arguments.length=0] > MemberExpression[property.name='toUpperCase']", }, ], - 'react/forbid-elements': [ - 1, - { - forbid: [{ element: 'button', message: 'use MUI