From 42addd8c8df87a0a93e9ae824acdf11f43e55030 Mon Sep 17 00:00:00 2001 From: Alex Rybchenko Date: Fri, 18 Nov 2022 14:16:58 +0100 Subject: [PATCH 001/437] added formData to validation function Signed-off-by: Alex Rybchenko --- plugins/scaffolder/api-report.md | 9 +++++---- plugins/scaffolder/src/extensions/types.ts | 3 ++- .../TemplateWizardPage/Stepper/createAsyncValidators.ts | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index 33a5f8c2fe..f94cb26d21 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -110,9 +110,9 @@ export type EntityPickerUiOptions = export const EntityTagsPickerFieldExtension: FieldExtensionComponent< string[], { - showCounts?: boolean | undefined; - kinds?: string[] | undefined; helperText?: string | undefined; + kinds?: string[] | undefined; + showCounts?: boolean | undefined; } >; @@ -120,9 +120,9 @@ export const EntityTagsPickerFieldExtension: FieldExtensionComponent< export const EntityTagsPickerFieldSchema: FieldSchema< string[], { - showCounts?: boolean | undefined; - kinds?: string[] | undefined; helperText?: string | undefined; + kinds?: string[] | undefined; + showCounts?: boolean | undefined; } >; @@ -224,6 +224,7 @@ export type NextCustomFieldValidator = ( field: FieldValidation_2, context: { apiHolder: ApiHolder; + formData: JsonObject; }, ) => void | Promise; diff --git a/plugins/scaffolder/src/extensions/types.ts b/plugins/scaffolder/src/extensions/types.ts index 3834b39c0a..5e133c676b 100644 --- a/plugins/scaffolder/src/extensions/types.ts +++ b/plugins/scaffolder/src/extensions/types.ts @@ -23,6 +23,7 @@ import { } from '@rjsf/utils'; import { PropsWithChildren } from 'react'; import { JSONSchema7 } from 'json-schema'; +import { JsonObject } from '@backstage/types'; /** * Field validation type for Custom Field Extensions. @@ -100,7 +101,7 @@ export interface NextFieldExtensionComponentProps< export type NextCustomFieldValidator = ( data: TFieldReturnValue, field: FieldValidationV5, - context: { apiHolder: ApiHolder }, + context: { apiHolder: ApiHolder; formData: JsonObject }, ) => void | Promise; /** diff --git a/plugins/scaffolder/src/next/TemplateWizardPage/Stepper/createAsyncValidators.ts b/plugins/scaffolder/src/next/TemplateWizardPage/Stepper/createAsyncValidators.ts index 1355387eda..6483ce2393 100644 --- a/plugins/scaffolder/src/next/TemplateWizardPage/Stepper/createAsyncValidators.ts +++ b/plugins/scaffolder/src/next/TemplateWizardPage/Stepper/createAsyncValidators.ts @@ -42,7 +42,7 @@ export const createAsyncValidators = ( if (validator) { const fieldValidation = createFieldValidation(); try { - await validator(value, fieldValidation, context); + await validator(value, fieldValidation, { ...context, formData }); } catch (ex) { fieldValidation.addError(ex.message); } From 596b7cf1a00729b06b80e967e6d2e0e834e213e9 Mon Sep 17 00:00:00 2001 From: Alex Rybchenko Date: Fri, 18 Nov 2022 14:47:10 +0100 Subject: [PATCH 002/437] initial support of ui options in scaffolder forms Signed-off-by: Alex Rybchenko --- .../sample-templates/bitbucket-demo/template.yaml | 2 ++ plugins/scaffolder-backend/src/service/router.ts | 1 + plugins/scaffolder/api-report.md | 3 +++ .../scaffolder/src/components/TemplatePage/TemplatePage.tsx | 1 + plugins/scaffolder/src/types.ts | 3 +++ 5 files changed, 10 insertions(+) diff --git a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml index 52714f6d73..269c3be1a6 100644 --- a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml @@ -4,6 +4,8 @@ metadata: name: bitbucket-demo title: Test Bitbucket RepoUrlPicker template description: scaffolder v1beta3 template demo publishing to bitbucket + ui:options: + finishButtonLabel: Publish spec: owner: backstage/techdocs-core type: service diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index eff7e95c50..be3c89601c 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -259,6 +259,7 @@ export async function createRouter( res.json({ title: template.metadata.title ?? template.metadata.name, description: template.metadata.description, + 'ui:options': template.metadata['ui:options'], steps: parameters.map(schema => ({ title: schema.title ?? 'Please enter the following information', description: schema.description, diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index 33a5f8c2fe..e9cbc5bb0e 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -640,6 +640,9 @@ export type TemplateGroupFilter = { export type TemplateParameterSchema = { title: string; description?: string; + ['ui:options']?: { + finishButtonLabel?: string; + }; steps: Array<{ title: string; description?: string; diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index 87678d2a83..4b2de4b56e 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -152,6 +152,7 @@ export const TemplatePage = ({ onReset={handleFormReset} onFinish={handleCreate} layouts={layouts} + finishButtonLabel={schema['ui:options']?.finishButtonLabel} steps={schema.steps.map(step => { return { ...step, diff --git a/plugins/scaffolder/src/types.ts b/plugins/scaffolder/src/types.ts index acb0f0e114..8645fc8697 100644 --- a/plugins/scaffolder/src/types.ts +++ b/plugins/scaffolder/src/types.ts @@ -80,6 +80,9 @@ export type ScaffolderTaskOutput = { export type TemplateParameterSchema = { title: string; description?: string; + ['ui:options']?: { + finishButtonLabel?: string; + }; steps: Array<{ title: string; description?: string; From b1ee86710147fac0b2383ef5aa7de956277ad44a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linn=C3=A9a=20Ivansson?= Date: Mon, 21 Nov 2022 14:42:01 +0100 Subject: [PATCH 003/437] Remove conditional rendering for ErrorEmptyState. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Linnéa Ivansson --- .../src/components/ErrorReporting/ErrorReporting.tsx | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx b/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx index 0dbf889fab..3bb1a81b1d 100644 --- a/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx +++ b/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx @@ -123,18 +123,14 @@ export const ErrorReporting = ({ detectedErrors }: ErrorReportingProps) => { return ( <> - {errors.length === 0 ? ( - - - - ) : ( + {errors.length !== 0 && - )} + } ); }; From 54017517f9ddc54e02e5fa6c4a18c6fada86beb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linn=C3=A9a=20Ivansson?= Date: Mon, 21 Nov 2022 14:49:34 +0100 Subject: [PATCH 004/437] Remove the no longer used ErrorEmptyState component. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Linnéa Ivansson --- .../ErrorReporting/ErrorReporting.tsx | 27 +------------------ 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx b/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx index 3bb1a81b1d..b87fa3b104 100644 --- a/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx +++ b/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx @@ -91,31 +91,6 @@ const sortBySeverity = (a: DetectedError, b: DetectedError) => { return 0; }; -export const ErrorEmptyState = () => { - return ( - - - - Nice! There are no errors to report! - - - - EmptyState - - - ); -}; - export const ErrorReporting = ({ detectedErrors }: ErrorReportingProps) => { const errors = Array.from(detectedErrors.values()) .flat() @@ -123,7 +98,7 @@ export const ErrorReporting = ({ detectedErrors }: ErrorReportingProps) => { return ( <> - {errors.length !== 0 && + {errors.length !== 0 &&
Date: Mon, 21 Nov 2022 14:54:29 +0100 Subject: [PATCH 005/437] Remove unused imports. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Linnéa Ivansson --- .../src/components/ErrorReporting/ErrorReporting.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx b/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx index b87fa3b104..6105f29ba8 100644 --- a/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx +++ b/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx @@ -15,9 +15,8 @@ */ import * as React from 'react'; import { DetectedError, DetectedErrorsByCluster } from '../../error-detection'; -import { Chip, Typography, Grid } from '@material-ui/core'; -import EmptyStateImage from '../../assets/emptystate.svg'; -import { Table, TableColumn, InfoCard } from '@backstage/core-components'; +import { Chip } from '@material-ui/core'; +import { Table, TableColumn } from '@backstage/core-components'; type ErrorReportingProps = { detectedErrors: DetectedErrorsByCluster; From 614083ef98be7297e3f4cf952e8f4f84acfdca13 Mon Sep 17 00:00:00 2001 From: Esther Annorzie Date: Mon, 21 Nov 2022 15:45:43 -0500 Subject: [PATCH 006/437] Add default message prop to StarredEntities Signed-off-by: Esther Annorzie --- .../homePageComponents/StarredEntities/Content.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/plugins/home/src/homePageComponents/StarredEntities/Content.tsx b/plugins/home/src/homePageComponents/StarredEntities/Content.tsx index 31fa00378e..aa0b6a5b72 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/Content.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/Content.tsx @@ -41,7 +41,14 @@ import useAsync from 'react-use/lib/useAsync'; * * @public */ -export const Content = () => { + +interface starredEntitiesProp { + defaultMessage?: string; +} + +export const Content = ({ + defaultMessage = 'Click the star beside an entity name to add the entity to this list!', +}: starredEntitiesProp) => { const catalogApi = useApi(catalogApiRef); const catalogEntityRoute = useRouteRef(entityRouteRef); const { starredEntities, toggleStarredEntity } = useStarredEntities(); @@ -76,7 +83,8 @@ export const Content = () => { if (starredEntities.size === 0) return ( - You do not have any starred entities yet! + {/* You do not have any starred entities yet! */} + {defaultMessage} ); From 9239c8acb87d2d95ca63f565208b4b03f07fa7c0 Mon Sep 17 00:00:00 2001 From: Esther Annorzie Date: Tue, 22 Nov 2022 10:09:55 -0500 Subject: [PATCH 007/437] Rename prop in starredEntitiesProp and remove comment Signed-off-by: Esther Annorzie --- .../src/homePageComponents/StarredEntities/Content.tsx | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/plugins/home/src/homePageComponents/StarredEntities/Content.tsx b/plugins/home/src/homePageComponents/StarredEntities/Content.tsx index aa0b6a5b72..1f31ac062f 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/Content.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/Content.tsx @@ -43,12 +43,10 @@ import useAsync from 'react-use/lib/useAsync'; */ interface starredEntitiesProp { - defaultMessage?: string; + noStarredEntitiesMessage?: React.ReactNode; } -export const Content = ({ - defaultMessage = 'Click the star beside an entity name to add the entity to this list!', -}: starredEntitiesProp) => { +export const Content = ({ noStarredEntitiesMessage }: starredEntitiesProp) => { const catalogApi = useApi(catalogApiRef); const catalogEntityRoute = useRouteRef(entityRouteRef); const { starredEntities, toggleStarredEntity } = useStarredEntities(); @@ -83,8 +81,8 @@ export const Content = ({ if (starredEntities.size === 0) return ( - {/* You do not have any starred entities yet! */} - {defaultMessage} + {noStarredEntitiesMessage || + 'Click the star beside an entity name to add it to this list!'} ); From b5e28f94fb8309b162fd50ecece32b937cd73a2b Mon Sep 17 00:00:00 2001 From: Esther Annorzie Date: Tue, 22 Nov 2022 18:05:10 -0500 Subject: [PATCH 008/437] Test call to action message when no entities are starred Signed-off-by: Esther Annorzie --- .../StarredEntities/Content.test.tsx | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx b/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx index 2b9fae8221..b761e89e6e 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx @@ -82,4 +82,34 @@ describe('StarredEntitiesContent', () => { '/catalog/default/component/mock-starred-entity-2', ); }); + + it('should display call to action message if no entities are starred', async () => { + const mockedApi = new MockStarredEntitiesApi(); + + const mockCatalogApi = { + getEntities: jest + .fn() + .mockImplementation(async () => ({ items: entities })), + }; + + const { getByText } = await renderInTestApp( + + + , + { + mountedRoutes: { + '/catalog/:namespace/:kind/:name': entityRouteRef, + }, + }, + ); + + expect( + getByText('Click the star beside an entity name to add it to this list!'), + ).toBeInTheDocument(); + }); }); From fd0b42cb27d5643e70d7c557de4c09a6686f352a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linn=C3=A9a=20Ivansson?= Date: Wed, 23 Nov 2022 10:17:48 +0100 Subject: [PATCH 009/437] Removed Devider generaly between grid items. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Linnéa Ivansson --- plugins/kubernetes/src/components/KubernetesContent.tsx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/plugins/kubernetes/src/components/KubernetesContent.tsx b/plugins/kubernetes/src/components/KubernetesContent.tsx index 3fabfae604..a42e9fbb0b 100644 --- a/plugins/kubernetes/src/components/KubernetesContent.tsx +++ b/plugins/kubernetes/src/components/KubernetesContent.tsx @@ -15,7 +15,7 @@ */ import React from 'react'; -import { Divider, Grid, Typography } from '@material-ui/core'; +import { Grid, Typography } from '@material-ui/core'; import { Entity } from '@backstage/catalog-model'; import { ErrorPanel } from './ErrorPanel'; import { ErrorReporting } from './ErrorReporting'; @@ -82,9 +82,6 @@ export const KubernetesContent = ({ - - - Your Clusters From 365f887717c9cdf3e503fc26e397bf0a6d8389aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linn=C3=A9a=20Ivansson?= Date: Wed, 23 Nov 2022 10:35:46 +0100 Subject: [PATCH 010/437] Add changeset. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Linnéa Ivansson --- .changeset/chilly-flies-nail.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/chilly-flies-nail.md diff --git a/.changeset/chilly-flies-nail.md b/.changeset/chilly-flies-nail.md new file mode 100644 index 0000000000..ffc9627b79 --- /dev/null +++ b/.changeset/chilly-flies-nail.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-kubernetes': patch +--- + +Removed rendering for ErrorEmptyState in ErrorReporting component, so nothing is rendered when there are no errors. Also removed Divider on Kubernetes page. From 777d6c7bdce0f1f741e32e6928a3caf0fe873873 Mon Sep 17 00:00:00 2001 From: Alex Rybchenko Date: Wed, 23 Nov 2022 15:53:36 +0100 Subject: [PATCH 011/437] leave only backend changes Signed-off-by: Alex Rybchenko --- .../sample-templates/bitbucket-demo/template.yaml | 2 -- plugins/scaffolder/api-report.md | 3 --- .../scaffolder/src/components/TemplatePage/TemplatePage.tsx | 1 - plugins/scaffolder/src/types.ts | 3 --- 4 files changed, 9 deletions(-) diff --git a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml index 269c3be1a6..52714f6d73 100644 --- a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml @@ -4,8 +4,6 @@ metadata: name: bitbucket-demo title: Test Bitbucket RepoUrlPicker template description: scaffolder v1beta3 template demo publishing to bitbucket - ui:options: - finishButtonLabel: Publish spec: owner: backstage/techdocs-core type: service diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index e9cbc5bb0e..33a5f8c2fe 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -640,9 +640,6 @@ export type TemplateGroupFilter = { export type TemplateParameterSchema = { title: string; description?: string; - ['ui:options']?: { - finishButtonLabel?: string; - }; steps: Array<{ title: string; description?: string; diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index 4b2de4b56e..87678d2a83 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -152,7 +152,6 @@ export const TemplatePage = ({ onReset={handleFormReset} onFinish={handleCreate} layouts={layouts} - finishButtonLabel={schema['ui:options']?.finishButtonLabel} steps={schema.steps.map(step => { return { ...step, diff --git a/plugins/scaffolder/src/types.ts b/plugins/scaffolder/src/types.ts index 8645fc8697..acb0f0e114 100644 --- a/plugins/scaffolder/src/types.ts +++ b/plugins/scaffolder/src/types.ts @@ -80,9 +80,6 @@ export type ScaffolderTaskOutput = { export type TemplateParameterSchema = { title: string; description?: string; - ['ui:options']?: { - finishButtonLabel?: string; - }; steps: Array<{ title: string; description?: string; From b07ccffad01f34abefa9cb3e5b313a9ab5a23e47 Mon Sep 17 00:00:00 2001 From: Alex Rybchenko Date: Wed, 23 Nov 2022 16:01:22 +0100 Subject: [PATCH 012/437] added changeset Signed-off-by: Alex Rybchenko --- .changeset/nasty-lizards-train.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/nasty-lizards-train.md diff --git a/.changeset/nasty-lizards-train.md b/.changeset/nasty-lizards-train.md new file mode 100644 index 0000000000..82d49cf682 --- /dev/null +++ b/.changeset/nasty-lizards-train.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Backend now returns 'ui:options' value from template metadata, it can be used by all your custom scaffolder components. From c75d80fdf0ae02a06a38d0a87b77954ebe5c7027 Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Fri, 11 Nov 2022 11:02:33 -0600 Subject: [PATCH 013/437] adding filters to catalog import Signed-off-by: Lucas De Souza --- .../src/components/ImportStepper/ImportStepper.tsx | 5 ++++- .../src/components/ImportStepper/defaults.tsx | 4 +++- .../StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx | 11 +++++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx index b2638e582e..7724a3c7d1 100644 --- a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx +++ b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx @@ -46,6 +46,7 @@ export interface ImportStepperProps { defaults: StepperProvider, ) => StepperProvider; variant?: InfoCardVariants; + filters?: Array; } /** @@ -58,6 +59,7 @@ export const ImportStepper = (props: ImportStepperProps) => { initialUrl, generateStepper = defaultGenerateStepper, variant, + filters = [], } = props; const catalogImportApi = useApi(catalogImportApiRef); @@ -88,7 +90,8 @@ export const ImportStepper = (props: ImportStepperProps) => { {render( states.analyze( state as Extract, - { apis: { catalogImportApi } }, + { apis: { catalogImportApi } }, + filters, ), )} {render( diff --git a/plugins/catalog-import/src/components/ImportStepper/defaults.tsx b/plugins/catalog-import/src/components/ImportStepper/defaults.tsx index 94dff39d0f..6d39a38e4e 100644 --- a/plugins/catalog-import/src/components/ImportStepper/defaults.tsx +++ b/plugins/catalog-import/src/components/ImportStepper/defaults.tsx @@ -51,6 +51,7 @@ export interface StepperProvider { analyze: ( s: Extract, opts: { apis: StepperApis }, + filters?: String[], ) => StepConfiguration; prepare: ( s: Extract, @@ -263,7 +264,7 @@ export function defaultGenerateStepper( } export const defaultStepper: StepperProvider = { - analyze: (state, { apis }) => ({ + analyze: (state, { apis }, filters=[]) => ({ stepLabel: Select URL, content: ( ), }), diff --git a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx index 3c655ec11b..51c5a4d3fd 100644 --- a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx +++ b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx @@ -42,6 +42,7 @@ export interface StepInitAnalyzeUrlProps { disablePullRequest?: boolean; analysisUrl?: string; exampleLocationUrl?: string; + filters?: Array, } /** @@ -58,6 +59,7 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { analysisUrl = '', disablePullRequest = false, exampleLocationUrl = 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + filters = [], } = props; const errorApi = useApi(errorApiRef); @@ -78,6 +80,8 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { const [submitted, setSubmitted] = useState(false); const [error, setError] = useState(undefined); + const filteredRegex = new RegExp(`^http[s]?://${filters ? `[${filters.map((filter, i) => i === 0 ? filter : `|${filter}`)}]` : ''}`) + const handleResult = useCallback( async ({ url }: FormData) => { setSubmitted(true); @@ -113,9 +117,8 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { } default: { - const err = `Received unknown analysis result of type ${ - (analysisResult as any).type - }. Please contact the support team.`; + const err = `Received unknown analysis result of type ${(analysisResult as any).type + }. Please contact the support team.`; setError(err); setSubmitted(false); @@ -140,7 +143,7 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { validate: { httpsValidator: (value: any) => (typeof value === 'string' && - value.match(/^http[s]?:\/\//) !== null) || + value.match(filteredRegex) !== null) || 'Must start with http:// or https://.', }, }), From 723dc1bb514c192b6557b2e4d78db59160f7857d Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Fri, 11 Nov 2022 11:13:41 -0600 Subject: [PATCH 014/437] adding filter to default import page Signed-off-by: Lucas De Souza --- .../components/DefaultImportPage/DefaultImportPage.tsx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx b/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx index 30cce9603a..0236e7c4d0 100644 --- a/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx +++ b/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx @@ -27,12 +27,17 @@ import React from 'react'; import { ImportInfoCard } from '../ImportInfoCard'; import { ImportStepper } from '../ImportStepper'; +interface DefaultImportPageProps { + filters?: Array; +} + /** * The default catalog import page. * * @public */ -export const DefaultImportPage = () => { +export const DefaultImportPage = (props: DefaultImportPageProps = { filters: [] }) => { + const { filters } = props; const configApi = useApi(configApiRef); const appTitle = configApi.getOptional('app.title') || 'Backstage'; @@ -53,7 +58,7 @@ export const DefaultImportPage = () => { - + From b4c84ca06bb2f97f6d364b4d70bff002d5a53442 Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Fri, 11 Nov 2022 11:19:54 -0600 Subject: [PATCH 015/437] adding filter to import page Signed-off-by: Lucas De Souza --- .../catalog-import/src/components/ImportPage/ImportPage.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx b/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx index 129e714952..beae0e5b3a 100644 --- a/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx +++ b/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx @@ -23,8 +23,8 @@ import { DefaultImportPage } from '../DefaultImportPage'; * * @public */ -export const ImportPage = () => { +export const ImportPage = ({ filters=[] }) => { const outlet = useOutlet(); - return outlet || ; + return outlet || ; }; From 8985c47ed2278cb0e253d158fc22d6142d8188c3 Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Tue, 15 Nov 2022 12:16:58 -0600 Subject: [PATCH 016/437] removing filter from component and add to rules Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- app-config.yaml | 3 +++ .../src/ingestion/CatalogRules.ts | 27 ++++++++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index dd72051189..3eb8e7e01c 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -218,6 +218,9 @@ catalog: - System - Domain - Location + - owners: + - Spotify + - Backstage processors: ldapOrg: diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 548c71d833..d25e936d1a 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -32,6 +32,9 @@ export type CatalogRule = { target?: string; type: string; }>; + owners?: Array<{ + owner: string + }>; }; /** @@ -94,6 +97,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (config.has('catalog.rules')) { const globalRules = config.getConfigArray('catalog.rules').map(sub => ({ allow: sub.getStringArray('allow').map(kind => ({ kind })), + owners: sub.getStringArray('owners').map(kind => ({ kind })), })); rules.push(...globalRules); } else { @@ -122,7 +126,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return new DefaultCatalogRulesEnforcer(rules); } - constructor(private readonly rules: CatalogRule[]) {} + constructor(private readonly rules: CatalogRule[]) { } /** * Checks whether a specific entity/location combination is allowed @@ -134,9 +138,13 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { continue; } - if (this.matchEntity(entity, rule.allow)) { - return true; + if (!this.matchOwners(entity, rule.owners)) { + return false; } + + if (this.matchEntity(entity, rule.allow)) { + return true; + } } return false; @@ -178,6 +186,19 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return false; } + + private matchOwners(entity: Entity, matchers?: { owner: string }[]): boolean { + if (!matchers) { + return true; + } + + const filteredRegex = new RegExp(`^http[s]?://${`(?:${matchers.map((filter, i) => i === 0 ? filter.owner : `|${filter.owner}`)})`}`); + + if ( entity?.metadata.links && entity?.metadata?.links?.length > 0) { + return filteredRegex.test(entity?.metadata?.links[0].url); + } + return false; + } } function resolveTarget(type: string, target: string): string { From ea96a0206429e6be499b750eba2e1704c0f5001e Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Tue, 15 Nov 2022 16:32:05 -0600 Subject: [PATCH 017/437] using sources instead of owners Signed-off-by: Lucas De Souza --- app-config.yaml | 3 --- .../src/ingestion/CatalogRules.ts | 19 ++++++++++--------- .../ImportStepper/ImportStepper.tsx | 2 -- .../src/components/ImportStepper/defaults.tsx | 3 +-- .../StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx | 5 +---- 5 files changed, 12 insertions(+), 20 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 3eb8e7e01c..dd72051189 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -218,9 +218,6 @@ catalog: - System - Domain - Location - - owners: - - Spotify - - Backstage processors: ldapOrg: diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index d25e936d1a..a48d29add6 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -32,8 +32,8 @@ export type CatalogRule = { target?: string; type: string; }>; - owners?: Array<{ - owner: string + sources?: Array<{ + source: string }>; }; @@ -58,6 +58,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { static readonly defaultRules: CatalogRule[] = [ { allow: ['Component', 'API', 'Location'].map(kind => ({ kind })), + sources: [], }, ]; @@ -97,7 +98,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (config.has('catalog.rules')) { const globalRules = config.getConfigArray('catalog.rules').map(sub => ({ allow: sub.getStringArray('allow').map(kind => ({ kind })), - owners: sub.getStringArray('owners').map(kind => ({ kind })), + sources: sub.getStringArray('sources').map(source => ({ source })), })); rules.push(...globalRules); } else { @@ -138,7 +139,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { continue; } - if (!this.matchOwners(entity, rule.owners)) { + if (!this.matchSources(location, rule.sources)) { return false; } @@ -187,15 +188,15 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return false; } - private matchOwners(entity: Entity, matchers?: { owner: string }[]): boolean { - if (!matchers) { + private matchSources(location: LocationSpec, matchers?: { source: string }[]): boolean { + if (!matchers || matchers.length === 0) { return true; } - const filteredRegex = new RegExp(`^http[s]?://${`(?:${matchers.map((filter, i) => i === 0 ? filter.owner : `|${filter.owner}`)})`}`); + const filteredRegex = new RegExp(`^http[s]?://${`(?:${matchers.map((filter, i) => i === 0 ? filter.source : `|${filter.source}`)})`}`); - if ( entity?.metadata.links && entity?.metadata?.links?.length > 0) { - return filteredRegex.test(entity?.metadata?.links[0].url); + if ( location.target && location.target.length > 0) { + return filteredRegex.test(location.target); } return false; } diff --git a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx index 7724a3c7d1..7352741326 100644 --- a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx +++ b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx @@ -59,7 +59,6 @@ export const ImportStepper = (props: ImportStepperProps) => { initialUrl, generateStepper = defaultGenerateStepper, variant, - filters = [], } = props; const catalogImportApi = useApi(catalogImportApiRef); @@ -91,7 +90,6 @@ export const ImportStepper = (props: ImportStepperProps) => { states.analyze( state as Extract, { apis: { catalogImportApi } }, - filters, ), )} {render( diff --git a/plugins/catalog-import/src/components/ImportStepper/defaults.tsx b/plugins/catalog-import/src/components/ImportStepper/defaults.tsx index 6d39a38e4e..d44a71dbdb 100644 --- a/plugins/catalog-import/src/components/ImportStepper/defaults.tsx +++ b/plugins/catalog-import/src/components/ImportStepper/defaults.tsx @@ -264,7 +264,7 @@ export function defaultGenerateStepper( } export const defaultStepper: StepperProvider = { - analyze: (state, { apis }, filters=[]) => ({ + analyze: (state, { apis }) => ({ stepLabel: Select URL, content: ( ), }), diff --git a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx index 51c5a4d3fd..49e4a4fa4a 100644 --- a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx +++ b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx @@ -59,7 +59,6 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { analysisUrl = '', disablePullRequest = false, exampleLocationUrl = 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', - filters = [], } = props; const errorApi = useApi(errorApiRef); @@ -80,8 +79,6 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { const [submitted, setSubmitted] = useState(false); const [error, setError] = useState(undefined); - const filteredRegex = new RegExp(`^http[s]?://${filters ? `[${filters.map((filter, i) => i === 0 ? filter : `|${filter}`)}]` : ''}`) - const handleResult = useCallback( async ({ url }: FormData) => { setSubmitted(true); @@ -143,7 +140,7 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { validate: { httpsValidator: (value: any) => (typeof value === 'string' && - value.match(filteredRegex) !== null) || + value.match(/^http[s]?:\/\//) !== null) || 'Must start with http:// or https://.', }, }), From 1b6bc7ca4c95fee525734899c8a89450b3805b31 Mon Sep 17 00:00:00 2001 From: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Date: Tue, 15 Nov 2022 16:40:20 -0600 Subject: [PATCH 018/437] Update plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx Co-authored-by: Lucas Desouza Signed-off-by: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Signed-off-by: Lucas De Souza --- .../src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx index 49e4a4fa4a..d936eaeba0 100644 --- a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx +++ b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx @@ -114,8 +114,9 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { } default: { - const err = `Received unknown analysis result of type ${(analysisResult as any).type - }. Please contact the support team.`; + const err = `Received unknown analysis result of type ${ + (analysisResult as any).type + }. Please contact the support team.` setError(err); setSubmitted(false); From 3a5d705fd70834077022614666581653d42186e0 Mon Sep 17 00:00:00 2001 From: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Date: Tue, 15 Nov 2022 16:40:31 -0600 Subject: [PATCH 019/437] Update plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx Co-authored-by: Lucas Desouza Signed-off-by: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Signed-off-by: Lucas De Souza --- .../src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx index d936eaeba0..aa9daffee1 100644 --- a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx +++ b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx @@ -42,7 +42,6 @@ export interface StepInitAnalyzeUrlProps { disablePullRequest?: boolean; analysisUrl?: string; exampleLocationUrl?: string; - filters?: Array, } /** From 430db47ad297ee16ad41d4997a4910eb80a73290 Mon Sep 17 00:00:00 2001 From: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Date: Tue, 15 Nov 2022 16:40:36 -0600 Subject: [PATCH 020/437] Update plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx Co-authored-by: Lucas Desouza Signed-off-by: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Signed-off-by: Lucas De Souza --- .../src/components/ImportStepper/ImportStepper.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx index 7352741326..ad1ad98465 100644 --- a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx +++ b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx @@ -89,7 +89,7 @@ export const ImportStepper = (props: ImportStepperProps) => { {render( states.analyze( state as Extract, - { apis: { catalogImportApi } }, + { apis: { catalogImportApi } }, ), )} {render( From cdbd1c000a3e2ce6a3ebd49d5121142964ed85d2 Mon Sep 17 00:00:00 2001 From: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Date: Tue, 15 Nov 2022 16:40:47 -0600 Subject: [PATCH 021/437] Update plugins/catalog-backend/src/ingestion/CatalogRules.ts Co-authored-by: Lucas Desouza Signed-off-by: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Signed-off-by: Lucas De Souza --- plugins/catalog-backend/src/ingestion/CatalogRules.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index a48d29add6..2c19773e41 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -33,7 +33,7 @@ export type CatalogRule = { type: string; }>; sources?: Array<{ - source: string + source: string; }>; }; From bec0c068c977f183005488c0181de9abb51180e2 Mon Sep 17 00:00:00 2001 From: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Date: Tue, 15 Nov 2022 16:40:54 -0600 Subject: [PATCH 022/437] Update plugins/catalog-backend/src/ingestion/CatalogRules.ts Co-authored-by: Lucas Desouza Signed-off-by: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Signed-off-by: Lucas De Souza --- plugins/catalog-backend/src/ingestion/CatalogRules.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 2c19773e41..903774b2f9 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -127,7 +127,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return new DefaultCatalogRulesEnforcer(rules); } - constructor(private readonly rules: CatalogRule[]) { } + constructor(private readonly rules: CatalogRule[]) {} /** * Checks whether a specific entity/location combination is allowed From 6b325f6b61eb66299974f7519bd1fc90326333a5 Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Tue, 15 Nov 2022 16:43:17 -0600 Subject: [PATCH 023/437] using optional string array Signed-off-by: Lucas De Souza --- app-config.yaml | 2 ++ plugins/catalog-backend/src/ingestion/CatalogRules.ts | 8 ++++---- .../components/DefaultImportPage/DefaultImportPage.tsx | 8 ++------ .../src/components/ImportStepper/defaults.tsx | 1 - 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index dd72051189..fd9263a5e8 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -218,6 +218,8 @@ catalog: - System - Domain - Location + sources: + - github.com/backstage processors: ldapOrg: diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 903774b2f9..398133036a 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -98,7 +98,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (config.has('catalog.rules')) { const globalRules = config.getConfigArray('catalog.rules').map(sub => ({ allow: sub.getStringArray('allow').map(kind => ({ kind })), - sources: sub.getStringArray('sources').map(source => ({ source })), + sources: (sub.getOptionalStringArray('sources') || []).map(source => ({ source })), })); rules.push(...globalRules); } else { @@ -143,9 +143,9 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return false; } - if (this.matchEntity(entity, rule.allow)) { - return true; - } + if (this.matchEntity(entity, rule.allow)) { + return true; + } } return false; diff --git a/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx b/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx index 0236e7c4d0..cec35574fc 100644 --- a/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx +++ b/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx @@ -27,17 +27,13 @@ import React from 'react'; import { ImportInfoCard } from '../ImportInfoCard'; import { ImportStepper } from '../ImportStepper'; -interface DefaultImportPageProps { - filters?: Array; -} /** * The default catalog import page. * * @public */ -export const DefaultImportPage = (props: DefaultImportPageProps = { filters: [] }) => { - const { filters } = props; +export const DefaultImportPage = () => { const configApi = useApi(configApiRef); const appTitle = configApi.getOptional('app.title') || 'Backstage'; @@ -58,7 +54,7 @@ export const DefaultImportPage = (props: DefaultImportPageProps = { filters: [] - + diff --git a/plugins/catalog-import/src/components/ImportStepper/defaults.tsx b/plugins/catalog-import/src/components/ImportStepper/defaults.tsx index d44a71dbdb..94dff39d0f 100644 --- a/plugins/catalog-import/src/components/ImportStepper/defaults.tsx +++ b/plugins/catalog-import/src/components/ImportStepper/defaults.tsx @@ -51,7 +51,6 @@ export interface StepperProvider { analyze: ( s: Extract, opts: { apis: StepperApis }, - filters?: String[], ) => StepConfiguration; prepare: ( s: Extract, From 08c58453d513329146559c46168ddb67866070f0 Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Tue, 15 Nov 2022 16:46:15 -0600 Subject: [PATCH 024/437] removing old filter Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- .../src/components/DefaultImportPage/DefaultImportPage.tsx | 1 - .../catalog-import/src/components/ImportPage/ImportPage.tsx | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx b/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx index cec35574fc..30cce9603a 100644 --- a/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx +++ b/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx @@ -27,7 +27,6 @@ import React from 'react'; import { ImportInfoCard } from '../ImportInfoCard'; import { ImportStepper } from '../ImportStepper'; - /** * The default catalog import page. * diff --git a/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx b/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx index beae0e5b3a..129e714952 100644 --- a/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx +++ b/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx @@ -23,8 +23,8 @@ import { DefaultImportPage } from '../DefaultImportPage'; * * @public */ -export const ImportPage = ({ filters=[] }) => { +export const ImportPage = () => { const outlet = useOutlet(); - return outlet || ; + return outlet || ; }; From bc405c9e37d5ab68434f20b1b499d7c1b5e59766 Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Tue, 15 Nov 2022 16:50:13 -0600 Subject: [PATCH 025/437] removing space Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- .../src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx index aa9daffee1..3c655ec11b 100644 --- a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx +++ b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx @@ -114,8 +114,8 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { default: { const err = `Received unknown analysis result of type ${ - (analysisResult as any).type - }. Please contact the support team.` + (analysisResult as any).type + }. Please contact the support team.`; setError(err); setSubmitted(false); From b711d136df6b8e13e087b5618bf789cc616b40fd Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Tue, 15 Nov 2022 16:51:29 -0600 Subject: [PATCH 026/437] remove filter in import stepper Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- .../src/components/ImportStepper/ImportStepper.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx index ad1ad98465..b2638e582e 100644 --- a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx +++ b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx @@ -46,7 +46,6 @@ export interface ImportStepperProps { defaults: StepperProvider, ) => StepperProvider; variant?: InfoCardVariants; - filters?: Array; } /** From f75ec75d3e38790f6957ac27fc4975181b628480 Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Thu, 17 Nov 2022 20:37:11 -0600 Subject: [PATCH 027/437] adding tests Signed-off-by: Lucas De Souza --- .../src/ingestion/CatalogRules.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index 2592b8b94b..43c3cd8f2f 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -35,6 +35,10 @@ const entity = { }; const location: Record = { + w: { + type: 'url', + target: 'https://github.com/backstage/blob/master/w.yaml', + }, x: { type: 'url', target: 'https://github.com/a/b/blob/master/x.yaml', @@ -216,5 +220,18 @@ describe('DefaultCatalogRulesEnforcer', () => { expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); expect(enforcer.isAllowed(entity.location, location.z)).toBe(false); }); + + it('should only allow sources that are specified in sources', () => { + const enforcer = DefaultCatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { + rules: [{ allow: ['Component'], sources: ['github.com/backstage'] }], + }, + }), + ); + expect(enforcer.isAllowed(entity.component, location.w)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + }); }); }); From e8303e99ff5bf7518673b0410feabce67c461825 Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 21 Nov 2022 15:01:30 -0600 Subject: [PATCH 028/437] updating to use location instead of sources Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- app-config.yaml | 3 +- .../src/ingestion/CatalogRules.test.ts | 8 ++-- .../src/ingestion/CatalogRules.ts | 46 ++++++++----------- 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index fd9263a5e8..d68391717e 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -218,8 +218,7 @@ catalog: - System - Domain - Location - sources: - - github.com/backstage + processors: ldapOrg: diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index 43c3cd8f2f..c9e0f7c619 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -37,7 +37,7 @@ const entity = { const location: Record = { w: { type: 'url', - target: 'https://github.com/backstage/blob/master/w.yaml', + target: 'https://github.com/b/c/blob/master/w.yaml', }, x: { type: 'url', @@ -209,7 +209,7 @@ describe('DefaultCatalogRulesEnforcer', () => { const enforcer = DefaultCatalogRulesEnforcer.fromConfig( new ConfigReader({ catalog: { - rules: [{ allow: ['Group'], locations: [{ type: 'url' }] }], + rules: [{ allow: ['Group'] }], }, }), ); @@ -221,11 +221,11 @@ describe('DefaultCatalogRulesEnforcer', () => { expect(enforcer.isAllowed(entity.location, location.z)).toBe(false); }); - it('should only allow sources that are specified in sources', () => { + it('should only allow locations that match a given regex', () => { const enforcer = DefaultCatalogRulesEnforcer.fromConfig( new ConfigReader({ catalog: { - rules: [{ allow: ['Component'], sources: ['github.com/backstage'] }], + rules: [{ allow: ['Component'], locations: [{type: 'url', match: 'https://github.com/b/*'}] }], }, }), ); diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 398133036a..43d865c4d2 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -31,9 +31,7 @@ export type CatalogRule = { locations?: Array<{ target?: string; type: string; - }>; - sources?: Array<{ - source: string; + match?: string; }>; }; @@ -58,7 +56,6 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { static readonly defaultRules: CatalogRule[] = [ { allow: ['Component', 'API', 'Location'].map(kind => ({ kind })), - sources: [], }, ]; @@ -96,10 +93,21 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { const rules = new Array(); if (config.has('catalog.rules')) { - const globalRules = config.getConfigArray('catalog.rules').map(sub => ({ - allow: sub.getStringArray('allow').map(kind => ({ kind })), - sources: (sub.getOptionalStringArray('sources') || []).map(source => ({ source })), - })); + const globalRules = config.getConfigArray('catalog.rules').map(ruleConfig => { + const rule: CatalogRule = { + allow: ruleConfig.getStringArray('allow').map(kind => ({ kind })), + }; + + const locConf = ruleConfig.getOptionalConfigArray('locations'); + if (locConf) + rule.locations = locConf.map( locationConfig => ({ + match: locationConfig.getOptionalString('match'), + type: locationConfig.getString('type'), + target: locationConfig.getOptionalString('target') + })) + + return rule; + }); rules.push(...globalRules); } else { rules.push(...DefaultCatalogRulesEnforcer.defaultRules); @@ -139,10 +147,6 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { continue; } - if (!this.matchSources(location, rule.sources)) { - return false; - } - if (this.matchEntity(entity, rule.allow)) { return true; } @@ -153,7 +157,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { private matchLocation( location: LocationSpec, - matchers?: { target?: string; type: string }[], + matchers?: { target?: string; type: string, match?: string }[], ): boolean { if (!matchers) { return true; @@ -166,6 +170,9 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (matcher.target && matcher.target !== location?.target) { continue; } + if (matcher.match && !location?.target.match(matcher.match)) { + continue; + } return true; } @@ -187,19 +194,6 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return false; } - - private matchSources(location: LocationSpec, matchers?: { source: string }[]): boolean { - if (!matchers || matchers.length === 0) { - return true; - } - - const filteredRegex = new RegExp(`^http[s]?://${`(?:${matchers.map((filter, i) => i === 0 ? filter.source : `|${filter.source}`)})`}`); - - if ( location.target && location.target.length > 0) { - return filteredRegex.test(location.target); - } - return false; - } } function resolveTarget(type: string, target: string): string { From c13544e033179e4da2c6bf75c884b285eedb371d Mon Sep 17 00:00:00 2001 From: Lucas Desouza Date: Mon, 21 Nov 2022 15:15:57 -0600 Subject: [PATCH 029/437] remove extra new line Co-authored-by: Zeky Abubaker Signed-off-by: Lucas Desouza Signed-off-by: Lucas De Souza --- app-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/app-config.yaml b/app-config.yaml index d68391717e..dd72051189 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -219,7 +219,6 @@ catalog: - Domain - Location - processors: ldapOrg: ### Example for how to add your enterprise LDAP server From 250b6f04f804317459ed098d139062466e5ceca8 Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 21 Nov 2022 15:40:56 -0600 Subject: [PATCH 030/437] run prettier Signed-off-by: Lucas De Souza --- app-config.yaml | 12 ++++++------ .../src/ingestion/CatalogRules.test.ts | 2 +- .../src/ingestion/CatalogRules.ts | 16 ++++++++-------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index dd72051189..c0f5767a45 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -212,12 +212,12 @@ catalog: pullRequestBranchName: backstage-integration rules: - allow: - - Component - - API - - Resource - - System - - Domain - - Location + - Component + - API + - Resource + - System + - Domain + - Location processors: ldapOrg: diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index c9e0f7c619..f4b9f5ea59 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -225,7 +225,7 @@ describe('DefaultCatalogRulesEnforcer', () => { const enforcer = DefaultCatalogRulesEnforcer.fromConfig( new ConfigReader({ catalog: { - rules: [{ allow: ['Component'], locations: [{type: 'url', match: 'https://github.com/b/*'}] }], + rules: [{ allow: ['Component'], locations: [{ type: 'url', match: 'https://github.com/b/*' }] }], }, }), ); diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 43d865c4d2..0d013dc78b 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -99,13 +99,13 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { }; const locConf = ruleConfig.getOptionalConfigArray('locations'); - if (locConf) - rule.locations = locConf.map( locationConfig => ({ - match: locationConfig.getOptionalString('match'), - type: locationConfig.getString('type'), - target: locationConfig.getOptionalString('target') - })) - + if (locConf) + rule.locations = locConf.map(locationConfig => ({ + match: locationConfig.getOptionalString('match'), + type: locationConfig.getString('type'), + target: locationConfig.getOptionalString('target') + })) + return rule; }); rules.push(...globalRules); @@ -135,7 +135,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return new DefaultCatalogRulesEnforcer(rules); } - constructor(private readonly rules: CatalogRule[]) {} + constructor(private readonly rules: CatalogRule[]) { } /** * Checks whether a specific entity/location combination is allowed From b6e8b6a23f16a30f71084e02e455cbdacfff122d Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Tue, 22 Nov 2022 10:19:45 -0600 Subject: [PATCH 031/437] run prettier Signed-off-by: Lucas De Souza --- app-config.yaml | 12 +++---- .../src/ingestion/CatalogRules.test.ts | 7 +++- .../src/ingestion/CatalogRules.ts | 32 ++++++++++--------- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index c0f5767a45..dd72051189 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -212,12 +212,12 @@ catalog: pullRequestBranchName: backstage-integration rules: - allow: - - Component - - API - - Resource - - System - - Domain - - Location + - Component + - API + - Resource + - System + - Domain + - Location processors: ldapOrg: diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index f4b9f5ea59..6cf8682129 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -225,7 +225,12 @@ describe('DefaultCatalogRulesEnforcer', () => { const enforcer = DefaultCatalogRulesEnforcer.fromConfig( new ConfigReader({ catalog: { - rules: [{ allow: ['Component'], locations: [{ type: 'url', match: 'https://github.com/b/*' }] }], + rules: [ + { + allow: ['Component'], + locations: [{ type: 'url', match: 'https://github.com/b/*' }], + }, + ], }, }), ); diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 0d013dc78b..66fe3da54e 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -93,21 +93,23 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { const rules = new Array(); if (config.has('catalog.rules')) { - const globalRules = config.getConfigArray('catalog.rules').map(ruleConfig => { - const rule: CatalogRule = { - allow: ruleConfig.getStringArray('allow').map(kind => ({ kind })), - }; + const globalRules = config + .getConfigArray('catalog.rules') + .map(ruleConfig => { + const rule: CatalogRule = { + allow: ruleConfig.getStringArray('allow').map(kind => ({ kind })), + }; - const locConf = ruleConfig.getOptionalConfigArray('locations'); - if (locConf) - rule.locations = locConf.map(locationConfig => ({ - match: locationConfig.getOptionalString('match'), - type: locationConfig.getString('type'), - target: locationConfig.getOptionalString('target') - })) + const locConf = ruleConfig.getOptionalConfigArray('locations'); + if (locConf) + rule.locations = locConf.map(locationConfig => ({ + match: locationConfig.getOptionalString('match'), + type: locationConfig.getString('type'), + target: locationConfig.getOptionalString('target'), + })); - return rule; - }); + return rule; + }); rules.push(...globalRules); } else { rules.push(...DefaultCatalogRulesEnforcer.defaultRules); @@ -135,7 +137,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return new DefaultCatalogRulesEnforcer(rules); } - constructor(private readonly rules: CatalogRule[]) { } + constructor(private readonly rules: CatalogRule[]) {} /** * Checks whether a specific entity/location combination is allowed @@ -157,7 +159,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { private matchLocation( location: LocationSpec, - matchers?: { target?: string; type: string, match?: string }[], + matchers?: { target?: string; type: string; match?: string }[], ): boolean { if (!matchers) { return true; From ba13ff663c930f563519c8f65bcceb01f918158a Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Tue, 22 Nov 2022 15:29:42 -0600 Subject: [PATCH 032/437] adding changeset Signed-off-by: Justin De Burgo --- .changeset/breezy-apes-mate.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/breezy-apes-mate.md diff --git a/.changeset/breezy-apes-mate.md b/.changeset/breezy-apes-mate.md new file mode 100644 index 0000000000..ac3409bd69 --- /dev/null +++ b/.changeset/breezy-apes-mate.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Adding an optional restriction for locations added through catalog import. Restrictions can be added using the app-config.yaml From 4c84c3d6ee61ccbde9e7eb16ef855cc4527a1c1a Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Wed, 23 Nov 2022 09:32:15 -0600 Subject: [PATCH 033/437] update config.d with new fields Signed-off-by: Lucas De Souza --- plugins/catalog-backend/config.d.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index daa70656bc..16094af489 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -38,6 +38,32 @@ export interface Config { * E.g. ["Component", "API", "Template", "Location"] */ allow: Array; + /** + * Limit this rule to a specific location + * + * Example with a fixed location + * { "type": "url", "target": "https://github.com/a/b/blob/file.yaml} + * + * Example using a Regex + * { "type": "url", "match": "https://github.com/a/*} + * + */ + location?: Array<{ + /** + * The type of location, e.g. "url". + */ + type: string; + /** + * The target URL of the location, e.g. + * "https://github.com/org/repo/blob/master/users.yaml". + */ + target?: string; + /** + * The target Regex of the location, e.g. + * "https://github.com/org/*. + */ + match?: string; + }>; }>; /** From da01dfdacd6b979cebb567d0256d463aa6917c0c Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 28 Nov 2022 12:04:28 -0600 Subject: [PATCH 034/437] use minimatch instead of regex Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- plugins/catalog-backend/config.d.ts | 6 +++--- plugins/catalog-backend/package.json | 1 + plugins/catalog-backend/src/ingestion/CatalogRules.test.ts | 2 +- plugins/catalog-backend/src/ingestion/CatalogRules.ts | 3 ++- yarn.lock | 1 + 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 16094af489..40d423d0d4 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -48,7 +48,7 @@ export interface Config { * { "type": "url", "match": "https://github.com/a/*} * */ - location?: Array<{ + locations?: Array<{ /** * The type of location, e.g. "url". */ @@ -59,8 +59,8 @@ export interface Config { */ target?: string; /** - * The target Regex of the location, e.g. - * "https://github.com/org/*. + * The pattern allowed for the location, e.g. + * "https://github.com/org/*\/blob/master/*.yaml. */ match?: string; }>; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index ea61ca648e..7c26d21a5b 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -59,6 +59,7 @@ "knex": "^2.0.0", "lodash": "^4.17.21", "luxon": "^3.0.0", + "minimatch": "^5.0.0", "node-fetch": "^2.6.7", "p-limit": "^3.0.2", "prom-client": "^14.0.1", diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index 6cf8682129..9432f1df1a 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -228,7 +228,7 @@ describe('DefaultCatalogRulesEnforcer', () => { rules: [ { allow: ['Component'], - locations: [{ type: 'url', match: 'https://github.com/b/*' }], + locations: [{ type: 'url', match: 'https://github.com/b/**' }], }, ], }, diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 66fe3da54e..3cda33533b 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -18,6 +18,7 @@ import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; import path from 'path'; import { LocationSpec } from '@backstage/plugin-catalog-common'; +import minimatch from 'minimatch'; /** * Rules to apply to catalog entities. @@ -172,7 +173,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (matcher.target && matcher.target !== location?.target) { continue; } - if (matcher.match && !location?.target.match(matcher.match)) { + if (matcher.match && !minimatch(location?.target, matcher.match, { nocase: true })) { continue; } return true; diff --git a/yarn.lock b/yarn.lock index 7537c367f6..49f1f4d5d4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5236,6 +5236,7 @@ __metadata: knex: ^2.0.0 lodash: ^4.17.21 luxon: ^3.0.0 + minimatch: ^5.0.0 msw: ^0.49.0 node-fetch: ^2.6.7 p-limit: ^3.0.2 From ac87547571f7055446e94ebf8ed0377f0a78cf7f Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 28 Nov 2022 12:06:10 -0600 Subject: [PATCH 035/437] update regex to pattern on test name Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- plugins/catalog-backend/src/ingestion/CatalogRules.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index 9432f1df1a..e87296e0a1 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -221,7 +221,7 @@ describe('DefaultCatalogRulesEnforcer', () => { expect(enforcer.isAllowed(entity.location, location.z)).toBe(false); }); - it('should only allow locations that match a given regex', () => { + it('should only allow locations that match a given pattern', () => { const enforcer = DefaultCatalogRulesEnforcer.fromConfig( new ConfigReader({ catalog: { From b285e53d4a747a666f4d8eb554ee70561d5fb23b Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 28 Nov 2022 12:16:00 -0600 Subject: [PATCH 036/437] cleanup Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- .../src/ingestion/CatalogRules.ts | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 3cda33533b..d4eda1db7e 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -96,21 +96,14 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (config.has('catalog.rules')) { const globalRules = config .getConfigArray('catalog.rules') - .map(ruleConfig => { - const rule: CatalogRule = { - allow: ruleConfig.getStringArray('allow').map(kind => ({ kind })), - }; - - const locConf = ruleConfig.getOptionalConfigArray('locations'); - if (locConf) - rule.locations = locConf.map(locationConfig => ({ + .map(ruleConf => ({ + allow: ruleConf.getStringArray('allow').map(kind => ({ kind })), + locations: ruleConf.getOptionalConfigArray('locations')?.map(locationConfig => ({ match: locationConfig.getOptionalString('match'), type: locationConfig.getString('type'), - target: locationConfig.getOptionalString('target'), - })); - - return rule; - }); + target: locationConfig.getOptionalString('target') + })) + })); rules.push(...globalRules); } else { rules.push(...DefaultCatalogRulesEnforcer.defaultRules); From 5cb15b7370927cf081c0bbacf0c869f00e19679d Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 28 Nov 2022 12:19:21 -0600 Subject: [PATCH 037/437] prettier Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- .../src/ingestion/CatalogRules.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index d4eda1db7e..3428eeaabc 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -97,13 +97,15 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { const globalRules = config .getConfigArray('catalog.rules') .map(ruleConf => ({ - allow: ruleConf.getStringArray('allow').map(kind => ({ kind })), - locations: ruleConf.getOptionalConfigArray('locations')?.map(locationConfig => ({ + allow: ruleConf.getStringArray('allow').map(kind => ({ kind })), + locations: ruleConf + .getOptionalConfigArray('locations') + ?.map(locationConfig => ({ match: locationConfig.getOptionalString('match'), type: locationConfig.getString('type'), - target: locationConfig.getOptionalString('target') - })) - })); + target: locationConfig.getOptionalString('target'), + })), + })); rules.push(...globalRules); } else { rules.push(...DefaultCatalogRulesEnforcer.defaultRules); @@ -166,7 +168,10 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (matcher.target && matcher.target !== location?.target) { continue; } - if (matcher.match && !minimatch(location?.target, matcher.match, { nocase: true })) { + if ( + matcher.match && + !minimatch(location?.target, matcher.match, { nocase: true }) + ) { continue; } return true; From cdb670ce5270f8171792eb0aa8d163dcd66090ea Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 28 Nov 2022 14:43:55 -0600 Subject: [PATCH 038/437] ensure that match and target are not both used Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- .../src/ingestion/CatalogRules.test.ts | 22 +++++++++++++++++++ .../src/ingestion/CatalogRules.ts | 18 ++++++++++----- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index e87296e0a1..06bb1389a9 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -54,6 +54,28 @@ const location: Record = { }; describe('DefaultCatalogRulesEnforcer', () => { + it('should throw an error if both match and target are used', () => { + expect(() => + DefaultCatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { + rules: [ + { + allow: ['Component'], + locations: [ + { + type: 'url', + match: 'https://github.com/b/**', + target: 'https://github.com/a/b/blob/master/w.yaml', + }, + ], + }, + ], + }, + }), + ), + ).toThrow(/cannot have both target and match values/i); + }); it('should deny by default', () => { const enforcer = new DefaultCatalogRulesEnforcer([]); expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 3428eeaabc..38561e0970 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -100,11 +100,19 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { allow: ruleConf.getStringArray('allow').map(kind => ({ kind })), locations: ruleConf .getOptionalConfigArray('locations') - ?.map(locationConfig => ({ - match: locationConfig.getOptionalString('match'), - type: locationConfig.getString('type'), - target: locationConfig.getOptionalString('target'), - })), + ?.map(locationConfig => { + const location = { + match: locationConfig.getOptionalString('match'), + type: locationConfig.getString('type'), + target: locationConfig.getOptionalString('target'), + }; + if (location.match && location.target) { + throw new Error( + 'A catalog rule location cannot have both target and match values', + ); + } + return location; + }), })); rules.push(...globalRules); } else { From 13278732f6e42b4991c993eda629f0cc9cf8cb03 Mon Sep 17 00:00:00 2001 From: Clare Liguori Date: Thu, 27 Oct 2022 13:10:41 -0700 Subject: [PATCH 039/437] New package for AWS integration node library Signed-off-by: Clare Liguori --- .changeset/forty-carpets-refuse.md | 5 + packages/integration-aws-node/.eslintrc.js | 1 + packages/integration-aws-node/README.md | 157 +++++++ packages/integration-aws-node/api-report.md | 73 +++ packages/integration-aws-node/config.d.ts | 123 +++++ packages/integration-aws-node/package.json | 55 +++ .../src/DefaultAwsCredentialsProvider.test.ts | 430 ++++++++++++++++++ .../src/DefaultAwsCredentialsProvider.ts | 274 +++++++++++ .../integration-aws-node/src/config.test.ts | 335 ++++++++++++++ packages/integration-aws-node/src/config.ts | 307 +++++++++++++ packages/integration-aws-node/src/index.ts | 29 ++ packages/integration-aws-node/src/types.ts | 57 +++ yarn.lock | 173 ++++++- 13 files changed, 2016 insertions(+), 3 deletions(-) create mode 100644 .changeset/forty-carpets-refuse.md create mode 100644 packages/integration-aws-node/.eslintrc.js create mode 100644 packages/integration-aws-node/README.md create mode 100644 packages/integration-aws-node/api-report.md create mode 100644 packages/integration-aws-node/config.d.ts create mode 100644 packages/integration-aws-node/package.json create mode 100644 packages/integration-aws-node/src/DefaultAwsCredentialsProvider.test.ts create mode 100644 packages/integration-aws-node/src/DefaultAwsCredentialsProvider.ts create mode 100644 packages/integration-aws-node/src/config.test.ts create mode 100644 packages/integration-aws-node/src/config.ts create mode 100644 packages/integration-aws-node/src/index.ts create mode 100644 packages/integration-aws-node/src/types.ts diff --git a/.changeset/forty-carpets-refuse.md b/.changeset/forty-carpets-refuse.md new file mode 100644 index 0000000000..a66121658d --- /dev/null +++ b/.changeset/forty-carpets-refuse.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration-aws-node': minor +--- + +New package for AWS integration node library diff --git a/packages/integration-aws-node/.eslintrc.js b/packages/integration-aws-node/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/packages/integration-aws-node/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/packages/integration-aws-node/README.md b/packages/integration-aws-node/README.md new file mode 100644 index 0000000000..6132f4240a --- /dev/null +++ b/packages/integration-aws-node/README.md @@ -0,0 +1,157 @@ +# @backstage/integration-aws-node + +This package providers helpers for fetching AWS account credentials +to be used by AWS SDK clients in backend packages and plugins. + +## Backstage app configuration + +Users of plugins and packages that use this library +will configure their AWS account information and credentials in their +Backstage app config. +Users can configure IAM user credentials, IAM roles, and profile names +for their AWS accounts in their Backstage config. + +If the AWS integration configuration is missing, the credentials provider +from this package will fall back to the AWS SDK default credentials chain for +resources in the main AWS account. +The default credentials chain for Node resolves credentials in the +following order of precedence: + +1. Environment variables +2. SSO credentials from token cache +3. Web identity token credentials +4. Shared credentials files +5. The EC2/ECS Instance Metadata Service + +See more about the AWS SDK default credentials chain in the +[AWS SDK for Javascript Developer Guide](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-credentials-node.html). + +Configuration examples: + +```yaml +aws: + # The main account is used as the source of credentials for calling + # the STS AssumeRole API to assume IAM roles in other AWS accounts. + # This section can be omitted to fall back to the AWS SDK's default creds chain. + mainAccount: + accessKeyId: ${MY_ACCESS_KEY_ID} + secretAccessKey: ${MY_SECRET_ACCESS_KEY} + + # Account credentials can be configured individually per account + accounts: + # Credentials can come from a role in the account + - accountId: '111111111111' + roleName: 'my-iam-role-name' + externalId: 'my-external-id' + + # Credentials can come from other AWS partitions + - accountId: '222222222222' + partition: 'aws-other' + roleName: 'my-iam-role-name' + # The STS region to use for the AssumeRole call + region: 'not-us-east-1' + # The creds to use when calling AssumeRole + accessKeyId: ${MY_ACCESS_KEY_ID_FOR_ANOTHER_PARTITION} + secretAccessKey: ${MY_SECRET_ACCESS_KEY_FOR_ANOTHER_PARTITION} + + # Credentials can come from static credentials + - accountId: '333333333333' + accessKeyId: ${MY_OTHER_ACCESS_KEY_ID} + secretAccessKey: ${MY_OTHER_SECRET_ACCESS_KEY} + + # Credentials can come from a profile in a shared config file on disk + - accountId: '444444444444' + profile: my-profile-name + + # Credentials can come from the AWS SDK's default creds chain + - accountId: '555555555555' + + # Credentials for accounts can fall back to a common role name. + # This is useful for account discovery use cases where the account + # IDs may not be known when writing the static config. + # If all accounts have a role with the same name, then the "accounts" + # section can be omitted entirely. + accountDefaults: + roleName: 'my-backstage-role' + externalId: 'my-id' +``` + +## Integrate new plugins + +Backend plugins can provide an AWS ARN or account ID to this library in order to +retrieve a credentials provider for the relevant account that can be fed directly +to an AWS SDK client. +The AWS SDK for Javascript V3 must be used. + +```typescript +const awsCredentialsProvider = DefaultAwsCredentialsProvider.fromConfig(config); + +// provide the account ID explicitly +const creds = await awsCredentialsProvider.getCredentials({ accountId }); +// OR extract the account ID from the ARN +const creds = await awsCredentialsProvider.getCredentials({ arn }); +// OR provide neither to get main account's credentials +const creds = await awsCredentialsProvider.getCredentials({}); + +// Example constructing an AWS Proton client with the returned credentials provider +const client = new ProtonClient({ + region, + credentialDefaultProvider: () => creds.provider, +}); +``` + +Depending on the nature of your plguin, you may either have the user specify the +relevant ARN or account ID in a catalog entity annotation or in the static Backstage +app configuration for your plugin. + +For example, you can create a new catalog entity annotation for your plugin: + +```yaml +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + annotations: + # Plugin annotation to specify an AWS account ID + my-plugin.io/aws-account-id: '123456789012' + # Plugin annotation to specify the AWS ARN of a specific resource + my-other-plugin.io/aws-dynamodb-table: 'arn:aws:dynamodb:us-east-2:123456789012:table/example-table' +``` + +In your plugin, read the annotation value so that you can retrieve the credentials provider: + +```typescript +const MY_AWS_ACCOUNT_ID_ANNOTATION = 'my-plugin.io/aws-account-id'; + +const getAwsAccountId = (entity: Entity) => + entity.metadata.annotations?.[MY_AWS_ACCOUNT_ID_ANNOTATION]); +``` + +Alternatively, you can create a new configuration field for your plugin: + +```yaml +# app-config.yaml +my-plugin: + # Statically configure the AWS account ID to use + awsAccountId: '123456789012' +my-other-plugin: + # Statically configure the AWS ARN of a specific resource + awsDynamoDbTable: 'arn:aws:dynamodb:us-east-2:123456789012:table/example-table' +``` + +In your plugin, read the configuration value so that you can retrieve the credentials provider: + +```typescript +// Read an account ID from your plugin's configuration +const awsCredentialsProvider = DefaultAwsCredentialsProvider.fromConfig(config); +const accountId = config.getString('my-plugin.awsAccountId'); +const creds = await awsCredentialsProvider.getCredentials({ accountId }); + +// Or, read an AWS ARN from your plugin's configuration +const awsCredentialsProvider = DefaultAwsCredentialsProvider.fromConfig(config); +const arn = config.getString('my-other-plugin.awsDynamoDbTable'); +const creds = await awsCredentialsProvider.getCredentials({ arn }); +``` + +## Links + +- [The Backstage homepage](https://backstage.io) diff --git a/packages/integration-aws-node/api-report.md b/packages/integration-aws-node/api-report.md new file mode 100644 index 0000000000..bac43c67d4 --- /dev/null +++ b/packages/integration-aws-node/api-report.md @@ -0,0 +1,73 @@ +## API Report File for "@backstage/integration-aws-node" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { AwsCredentialIdentityProvider } from '@aws-sdk/types'; +import { Config } from '@backstage/config'; + +// @public +export type AwsCredentials = { + accountId?: string; + stsRegion?: string; + provider: AwsCredentialIdentityProvider; +}; + +// @public +export interface AwsCredentialsProvider { + getCredentials(opts?: AwsCredentialsProviderOptions): Promise; +} + +// @public +export type AwsCredentialsProviderOptions = { + accountId?: string; + arn?: string; +}; + +// @public +export type AwsIntegrationAccountConfig = { + accountId: string; + accessKeyId?: string; + secretAccessKey?: string; + profile?: string; + roleName?: string; + partition?: string; + region?: string; + externalId?: string; +}; + +// @public +export type AwsIntegrationConfig = { + accounts: AwsIntegrationAccountConfig[]; + accountDefaults: AwsIntegrationDefaultAccountConfig; + mainAccount: AwsIntegrationMainAccountConfig; +}; + +// @public +export type AwsIntegrationDefaultAccountConfig = { + roleName?: string; + partition?: string; + region?: string; + externalId?: string; +}; + +// @public +export type AwsIntegrationMainAccountConfig = { + accessKeyId?: string; + secretAccessKey?: string; + profile?: string; + region?: string; +}; + +// @public +export class DefaultAwsCredentialsProvider implements AwsCredentialsProvider { + // (undocumented) + static fromConfig(config: Config): DefaultAwsCredentialsProvider; + getCredentials(opts?: AwsCredentialsProviderOptions): Promise; +} + +// @public +export function readAwsIntegrationConfig(config: Config): AwsIntegrationConfig; + +// (No @packageDocumentation comment for this package) +``` diff --git a/packages/integration-aws-node/config.d.ts b/packages/integration-aws-node/config.d.ts new file mode 100644 index 0000000000..3c5600efc3 --- /dev/null +++ b/packages/integration-aws-node/config.d.ts @@ -0,0 +1,123 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + /** Configuration for access to AWS accounts */ + aws?: { + /** + * Defaults for retrieving AWS account credentials + */ + accountDefaults?: { + /** + * The IAM role to assume to retrieve temporary AWS credentials + */ + roleName?: string; + + /** + * The AWS partition of the IAM role, e.g. "aws", "aws-cn" + */ + partition?: string; + + /** + * The STS regional endpoint to use when retrieving temporary AWS credentials, e.g. "ap-northeast-1" + */ + region?: string; + + /** + * The unique identifier needed to assume the role to retrieve temporary AWS credentials + * @visibility secret + */ + externalId?: string; + }; + + /** + * Main account to use for retrieving AWS account credentials + */ + mainAccount?: { + /** + * The access key ID for a set of static AWS credentials + * @visibility secret + */ + accessKeyId?: string; + + /** + * The secret access key for a set of static AWS credentials + * @visibility secret + */ + secretAccessKey?: string; + + /** + * The configuration profile from a credentials file at ~/.aws/credentials and + * a configuration file at ~/.aws/config. + */ + profile?: string; + + /** + * The STS regional endpoint to use for the main account, e.g. "ap-northeast-1" + */ + region?: string; + }; + + /** + * Configuration for retrieving AWS accounts credentials + */ + accounts?: Array<{ + /** + * The account ID of the target account that this matches on, e.g. "123456789012" + */ + accountId: string; + + /** + * The access key ID for a set of static AWS credentials + * @visibility secret + */ + accessKeyId?: string; + + /** + * The secret access key for a set of static AWS credentials + * @visibility secret + */ + secretAccessKey?: string; + + /** + * The configuration profile from a credentials file at ~/.aws/credentials and + * a configuration file at ~/.aws/config. + */ + profile?: string; + + /** + * The IAM role to assume to retrieve temporary AWS credentials + */ + roleName?: string; + + /** + * The AWS partition of the IAM role, e.g. "aws", "aws-cn" + */ + partition?: string; + + /** + * The STS regional endpoint to use when retrieving temporary AWS credentials, e.g. "ap-northeast-1" + */ + region?: string; + + /** + * The unique identifier needed to assume the role to retrieve temporary AWS credentials + * @visibility secret + */ + externalId?: string; + }>; + }; +} diff --git a/packages/integration-aws-node/package.json b/packages/integration-aws-node/package.json new file mode 100644 index 0000000000..b3560668b8 --- /dev/null +++ b/packages/integration-aws-node/package.json @@ -0,0 +1,55 @@ +{ + "name": "@backstage/integration-aws-node", + "description": "Helpers for fetching AWS account credentials", + "version": "0.0.0", + "main": "src/index.ts", + "types": "src/index.ts", + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "module": "dist/index.esm.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "node-library" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "packages/integration-aws-node" + }, + "keywords": [ + "backstage" + ], + "license": "Apache-2.0", + "scripts": { + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "clean": "backstage-cli package clean" + }, + "dependencies": { + "@aws-sdk/client-sts": "^3.208.0", + "@aws-sdk/credential-provider-node": "^3.208.0", + "@aws-sdk/credential-providers": "^3.208.0", + "@aws-sdk/types": "^3.208.0", + "@aws-sdk/util-arn-parser": "^3.208.0", + "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^" + }, + "devDependencies": { + "@backstage/cli": "workspace:^", + "@backstage/config-loader": "workspace:^", + "@backstage/test-utils": "workspace:^", + "aws-sdk-client-mock": "^2.0.0", + "aws-sdk-client-mock-jest": "^2.0.0" + }, + "files": [ + "dist", + "config.d.ts" + ], + "configSchema": "config.d.ts" +} diff --git a/packages/integration-aws-node/src/DefaultAwsCredentialsProvider.test.ts b/packages/integration-aws-node/src/DefaultAwsCredentialsProvider.test.ts new file mode 100644 index 0000000000..60f2f558b8 --- /dev/null +++ b/packages/integration-aws-node/src/DefaultAwsCredentialsProvider.test.ts @@ -0,0 +1,430 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DefaultAwsCredentialsProvider } from './DefaultAwsCredentialsProvider'; +import { mockClient, AwsClientStub } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest'; +import { + STSClient, + GetCallerIdentityCommand, + AssumeRoleCommand, +} from '@aws-sdk/client-sts'; +import { Config, ConfigReader } from '@backstage/config'; +import { promises } from 'fs'; + +const env = process.env; +let stsMock: AwsClientStub; +let config: Config; + +jest.mock('fs', () => ({ promises: { readFile: jest.fn() } })); + +describe('DefaultAwsCredentialsProvider', () => { + beforeEach(() => { + process.env = { ...env }; + jest.resetAllMocks(); + + stsMock = mockClient(STSClient); + + config = new ConfigReader({ + aws: { + accounts: [ + { + accountId: '111111111111', + roleName: 'hello', + externalId: 'world', + }, + { + accountId: '222222222222', + roleName: 'hi', + partition: 'aws-other', + region: 'not-us-east-1', + accessKeyId: 'ABC', + secretAccessKey: 'EDF', + }, + { + accountId: '333333333333', + accessKeyId: 'my-access-key', + secretAccessKey: 'my-secret-access-key', + }, + { + accountId: '444444444444', + }, + { + accountId: '555555555555', + profile: 'my-profile', + }, + ], + accountDefaults: { + roleName: 'backstage-role', + externalId: 'my-id', + }, + mainAccount: { + accessKeyId: 'GHI', + secretAccessKey: 'JKL', + region: 'ap-northeast-1', + }, + }, + }); + + stsMock.on(GetCallerIdentityCommand).resolvesOnce({ + Account: '123456789012', + }); + + stsMock + .on(AssumeRoleCommand, { + RoleArn: 'arn:aws:iam::111111111111:role/hello', + RoleSessionName: 'backstage', + ExternalId: 'world', + }) + .resolves({ + Credentials: { + AccessKeyId: 'ACCESS_KEY_ID_1', + SecretAccessKey: 'SECRET_ACCESS_KEY_1', + SessionToken: 'SESSION_TOKEN_1', + Expiration: new Date('2022-01-01'), + }, + }); + + stsMock + .on(AssumeRoleCommand, { + RoleArn: 'arn:aws-other:iam::222222222222:role/hi', + RoleSessionName: 'backstage', + }) + .resolves({ + Credentials: { + AccessKeyId: 'ACCESS_KEY_ID_2', + SecretAccessKey: 'SECRET_ACCESS_KEY_2', + SessionToken: 'SESSION_TOKEN_2', + Expiration: new Date('2022-01-02'), + }, + }); + + stsMock + .on(AssumeRoleCommand, { + RoleArn: 'arn:aws:iam::999999999999:role/backstage-role', + RoleSessionName: 'backstage', + ExternalId: 'my-id', + }) + .resolves({ + Credentials: { + AccessKeyId: 'ACCESS_KEY_ID_9', + SecretAccessKey: 'SECRET_ACCESS_KEY_9', + SessionToken: 'SESSION_TOKEN_9', + Expiration: new Date('2022-01-09'), + }, + }); + + process.env.AWS_ACCESS_KEY_ID = 'ACCESS_KEY_ID_10'; + process.env.AWS_SECRET_ACCESS_KEY = 'SECRET_ACCESS_KEY_10'; + process.env.AWS_SESSION_TOKEN = 'SESSION_TOKEN_10'; + process.env.AWS_CREDENTIAL_EXPIRATION = new Date( + '2022-01-10', + ).toISOString(); + + const mockProfile = `[my-profile] + aws_access_key_id=ACCESS_KEY_ID_9 + aws_secret_access_key=SECRET_ACCESS_KEY_9 + `; + (promises.readFile as jest.Mock).mockResolvedValue(mockProfile); + }); + + afterEach(() => { + process.env = env; + }); + + describe('#getCredentials', () => { + it('retrieves assume-role creds for the given account ID and caches the provider', async () => { + const provider = DefaultAwsCredentialsProvider.fromConfig(config); + const awsCredentials = await provider.getCredentials({ + accountId: '111111111111', + }); + + expect(awsCredentials.accountId).toEqual('111111111111'); + + const creds = await awsCredentials.provider(); + expect(creds).toEqual({ + accessKeyId: 'ACCESS_KEY_ID_1', + secretAccessKey: 'SECRET_ACCESS_KEY_1', + sessionToken: 'SESSION_TOKEN_1', + expiration: new Date('2022-01-01'), + }); + + const awsCredentials2 = await provider.getCredentials({ + accountId: '111111111111', + }); + + expect(awsCredentials).toBe(awsCredentials2); + expect(stsMock).toHaveReceivedCommandTimes(AssumeRoleCommand, 1); + }); + + it('retrieves assume-role creds in another partition for the given account ID', async () => { + const provider = DefaultAwsCredentialsProvider.fromConfig(config); + const awsCredentials = await provider.getCredentials({ + accountId: '222222222222', + }); + + expect(awsCredentials.accountId).toEqual('222222222222'); + + const creds = await awsCredentials.provider(); + expect(creds).toEqual({ + accessKeyId: 'ACCESS_KEY_ID_2', + secretAccessKey: 'SECRET_ACCESS_KEY_2', + sessionToken: 'SESSION_TOKEN_2', + expiration: new Date('2022-01-02'), + }); + }); + + it('retrieves assume-role creds for an account using the account defaults', async () => { + const provider = DefaultAwsCredentialsProvider.fromConfig(config); + const awsCredentials = await provider.getCredentials({ + accountId: '999999999999', + }); + + expect(awsCredentials.accountId).toEqual('999999999999'); + + const creds = await awsCredentials.provider(); + expect(creds).toEqual({ + accessKeyId: 'ACCESS_KEY_ID_9', + secretAccessKey: 'SECRET_ACCESS_KEY_9', + sessionToken: 'SESSION_TOKEN_9', + expiration: new Date('2022-01-09'), + }); + }); + + it('retrieves static creds for the given account ID', async () => { + const provider = DefaultAwsCredentialsProvider.fromConfig(config); + const awsCredentials = await provider.getCredentials({ + accountId: '333333333333', + }); + + expect(awsCredentials.accountId).toEqual('333333333333'); + + const creds = await awsCredentials.provider(); + expect(creds).toEqual({ + accessKeyId: 'my-access-key', + secretAccessKey: 'my-secret-access-key', + }); + }); + + it('retrieves static creds from the main account', async () => { + const minConfig = new ConfigReader({ + aws: { + mainAccount: { + accessKeyId: 'GHI', + secretAccessKey: 'JKL', + }, + }, + }); + const provider = DefaultAwsCredentialsProvider.fromConfig(minConfig); + const awsCredentials = await provider.getCredentials({ + accountId: '123456789012', + }); + + expect(awsCredentials.accountId).toEqual('123456789012'); + + const creds = await awsCredentials.provider(); + expect(creds).toEqual({ + accessKeyId: 'GHI', + secretAccessKey: 'JKL', + }); + }); + + it('only queries the main account ID once from STS', async () => { + const minConfig = new ConfigReader({ + aws: { + mainAccount: { + accessKeyId: 'GHI', + secretAccessKey: 'JKL', + }, + }, + }); + const provider = DefaultAwsCredentialsProvider.fromConfig(minConfig); + const awsCredentials1 = await provider.getCredentials({}); + const awsCredentials2 = await provider.getCredentials({}); + + expect(awsCredentials1).toBe(awsCredentials2); + expect(stsMock).toHaveReceivedCommandTimes(GetCallerIdentityCommand, 1); + }); + + it('retrieves the ini provider chain for the given account ID', async () => { + const provider = DefaultAwsCredentialsProvider.fromConfig(config); + const awsCredentials = await provider.getCredentials({ + accountId: '555555555555', + }); + + expect(awsCredentials.accountId).toEqual('555555555555'); + + const creds = await awsCredentials.provider(); + expect(creds).toEqual({ + accessKeyId: 'ACCESS_KEY_ID_9', + secretAccessKey: 'SECRET_ACCESS_KEY_9', + }); + }); + + it('retrieves the default cred provider chain for the given account ID', async () => { + const provider = DefaultAwsCredentialsProvider.fromConfig(config); + const awsCredentials = await provider.getCredentials({ + accountId: '444444444444', + }); + + expect(awsCredentials.accountId).toEqual('444444444444'); + + const creds = await awsCredentials.provider(); + expect(creds).toEqual({ + accessKeyId: 'ACCESS_KEY_ID_10', + secretAccessKey: 'SECRET_ACCESS_KEY_10', + sessionToken: 'SESSION_TOKEN_10', + expiration: new Date('2022-01-10'), + }); + }); + + it('retrieves ini provider chain from the main account', async () => { + const minConfig = new ConfigReader({ + aws: { + mainAccount: { + profile: 'my-profile', + }, + }, + }); + const provider = DefaultAwsCredentialsProvider.fromConfig(minConfig); + const awsCredentials = await provider.getCredentials({ + accountId: '123456789012', + }); + + expect(awsCredentials.accountId).toEqual('123456789012'); + + const creds = await awsCredentials.provider(); + expect(creds).toEqual({ + accessKeyId: 'ACCESS_KEY_ID_9', + secretAccessKey: 'SECRET_ACCESS_KEY_9', + }); + }); + + it('retrieves default cred provider chain from the main account', async () => { + const minConfig = new ConfigReader({ + aws: {}, + }); + const provider = DefaultAwsCredentialsProvider.fromConfig(minConfig); + const awsCredentials = await provider.getCredentials({ + accountId: '123456789012', + }); + + expect(awsCredentials.accountId).toEqual('123456789012'); + + const creds = await awsCredentials.provider(); + expect(creds).toEqual({ + accessKeyId: 'ACCESS_KEY_ID_10', + secretAccessKey: 'SECRET_ACCESS_KEY_10', + sessionToken: 'SESSION_TOKEN_10', + expiration: new Date('2022-01-10'), + }); + }); + + it('retrieves default cred provider chain from the main account when there is no AWS integration config', async () => { + const minConfig = new ConfigReader({}); + const provider = DefaultAwsCredentialsProvider.fromConfig(minConfig); + const awsCredentials = await provider.getCredentials({ + accountId: '123456789012', + }); + + expect(awsCredentials.accountId).toEqual('123456789012'); + + const creds = await awsCredentials.provider(); + expect(creds).toEqual({ + accessKeyId: 'ACCESS_KEY_ID_10', + secretAccessKey: 'SECRET_ACCESS_KEY_10', + sessionToken: 'SESSION_TOKEN_10', + expiration: new Date('2022-01-10'), + }); + }); + + it('extracts the account ID from an ARN', async () => { + const provider = DefaultAwsCredentialsProvider.fromConfig(config); + const awsCredentials = await provider.getCredentials({ + arn: 'arn:aws:ecs:region:111111111111:service/cluster-name/service-name', + }); + + expect(awsCredentials.accountId).toEqual('111111111111'); + + const creds = await awsCredentials.provider(); + expect(creds).toEqual({ + accessKeyId: 'ACCESS_KEY_ID_1', + secretAccessKey: 'SECRET_ACCESS_KEY_1', + sessionToken: 'SESSION_TOKEN_1', + expiration: new Date('2022-01-01'), + }); + }); + + it('falls back to main account credentials when account ID cannot be extracted from the ARN', async () => { + const provider = DefaultAwsCredentialsProvider.fromConfig(config); + const awsCredentials = await provider.getCredentials({ + arn: 'arn:aws:s3:::bucket_name', + }); + + expect(awsCredentials.accountId).toEqual('123456789012'); + + const creds = await awsCredentials.provider(); + expect(creds).toEqual({ + accessKeyId: 'GHI', + secretAccessKey: 'JKL', + }); + }); + + it('falls back to main account credentials when neither account ID nor ARN are provided', async () => { + const provider = DefaultAwsCredentialsProvider.fromConfig(config); + const awsCredentials = await provider.getCredentials({}); + + expect(awsCredentials.accountId).toEqual('123456789012'); + + const creds = await awsCredentials.provider(); + expect(creds).toEqual({ + accessKeyId: 'GHI', + secretAccessKey: 'JKL', + }); + }); + + it('falls back to main account credentials when no options are provided', async () => { + const provider = DefaultAwsCredentialsProvider.fromConfig(config); + const awsCredentials = await provider.getCredentials(); + + expect(awsCredentials.accountId).toEqual('123456789012'); + + const creds = await awsCredentials.provider(); + expect(creds).toEqual({ + accessKeyId: 'GHI', + secretAccessKey: 'JKL', + }); + }); + + it('rejects account that is not configured, with no account defaults', async () => { + const minConfig = new ConfigReader({ + aws: {}, + }); + const provider = DefaultAwsCredentialsProvider.fromConfig(minConfig); + await expect( + provider.getCredentials({ accountId: '111222333444' }), + ).rejects.toThrow(/no AWS integration that matches 111222333444/); + }); + + it('rejects main account that has invalid credentials', async () => { + stsMock.on(GetCallerIdentityCommand).rejects('No credentials found'); + const provider = DefaultAwsCredentialsProvider.fromConfig(config); + await expect(provider.getCredentials({})).rejects.toThrow( + /No credentials found/, + ); + }); + }); +}); diff --git a/packages/integration-aws-node/src/DefaultAwsCredentialsProvider.ts b/packages/integration-aws-node/src/DefaultAwsCredentialsProvider.ts new file mode 100644 index 0000000000..8b26b5fe84 --- /dev/null +++ b/packages/integration-aws-node/src/DefaultAwsCredentialsProvider.ts @@ -0,0 +1,274 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + readAwsIntegrationConfig, + AwsIntegrationAccountConfig, + AwsIntegrationDefaultAccountConfig, + AwsIntegrationMainAccountConfig, +} from './config'; +import { + AwsCredentials, + AwsCredentialsProvider, + AwsCredentialsProviderOptions, +} from './types'; +import { GetCallerIdentityCommand, STSClient } from '@aws-sdk/client-sts'; +import { + fromIni, + fromNodeProviderChain, + fromTemporaryCredentials, +} from '@aws-sdk/credential-providers'; +import { AwsCredentialIdentityProvider } from '@aws-sdk/types'; +import { parse } from '@aws-sdk/util-arn-parser'; +import { Config } from '@backstage/config'; + +/** + * Retrieves the account ID for the given credentials provider from STS. + */ +async function fillInAccountId(creds: AwsCredentials) { + if (creds.accountId) { + return; + } + + const client = new STSClient({ + region: creds.stsRegion, + customUserAgent: 'backstage-aws-credentials-provider', + credentialDefaultProvider: () => creds.provider, + }); + const resp = await client.send(new GetCallerIdentityCommand({})); + creds.accountId = resp.Account!; +} + +function getStaticCredentials( + accessKeyId: string, + secretAccessKey: string, +): AwsCredentialIdentityProvider { + return async () => { + return Promise.resolve({ + accessKeyId: accessKeyId, + secretAccessKey: secretAccessKey, + }); + }; +} + +function getProfileCredentials( + profile: string, + region?: string, +): AwsCredentialIdentityProvider { + return fromIni({ + profile, + clientConfig: { + region, + customUserAgent: 'backstage-aws-credentials-provider', + }, + }); +} + +function getDefaultCredentialsChain(): AwsCredentialIdentityProvider { + return fromNodeProviderChain(); +} + +/** + * Constructs the credential provider needed by the AWS SDK from the given account config + * + * Order of precedence: + * 1. Assume role with static creds + * 2. Assume role with main account creds + * 3. Static creds + * 4. Profile creds + * 5. Default AWS SDK creds chain + */ +function getAccountCredentialsProvider( + config: AwsIntegrationAccountConfig, + mainAccountCreds: AwsCredentialIdentityProvider, +): AwsCredentialIdentityProvider { + if (config.roleName) { + const region = config.region ?? 'us-east-1'; + const partition = config.partition ?? 'aws'; + + return fromTemporaryCredentials({ + masterCredentials: config.accessKeyId + ? getStaticCredentials(config.accessKeyId!, config.secretAccessKey!) + : mainAccountCreds, + params: { + RoleArn: `arn:${partition}:iam::${config.accountId}:role/${config.roleName}`, + RoleSessionName: 'backstage', + ExternalId: config.externalId, + }, + clientConfig: { + region, + customUserAgent: 'backstage-aws-credentials-provider', + }, + }); + } + + if (config.accessKeyId) { + return getStaticCredentials(config.accessKeyId!, config.secretAccessKey!); + } + + if (config.profile) { + return getProfileCredentials(config.profile!, config.region); + } + + return getDefaultCredentialsChain(); +} + +/** + * Constructs the credential provider needed by the AWS SDK for the main account + * + * Order of precedence: + * 1. Static creds + * 2. Profile creds + * 3. Default AWS SDK creds chain + */ +function getMainAccountCredentialsProvider( + config: AwsIntegrationMainAccountConfig, +): AwsCredentialIdentityProvider { + if (config.accessKeyId) { + return getStaticCredentials(config.accessKeyId!, config.secretAccessKey!); + } + + if (config.profile) { + return getProfileCredentials(config.profile!, config.region); + } + + return getDefaultCredentialsChain(); +} + +/** + * Handles the creation and caching of credential providers for AWS accounts. + * + * @public + */ +export class DefaultAwsCredentialsProvider implements AwsCredentialsProvider { + static fromConfig(config: Config): DefaultAwsCredentialsProvider { + const awsConfig = config.has('aws') + ? readAwsIntegrationConfig(config.getConfig('aws')) + : { + accounts: [], + mainAccount: {}, + accountDefaults: {}, + }; + + const mainAccountProvider = getMainAccountCredentialsProvider( + awsConfig.mainAccount, + ); + const mainAccountCreds: AwsCredentials = { + provider: mainAccountProvider, + }; + + const accountCreds = new Map(); + for (const accountConfig of awsConfig.accounts) { + const provider = getAccountCredentialsProvider( + accountConfig, + mainAccountCreds.provider, + ); + accountCreds.set(accountConfig.accountId, { + accountId: accountConfig.accountId, + stsRegion: accountConfig.region, + provider, + }); + } + + return new DefaultAwsCredentialsProvider( + accountCreds, + awsConfig.accountDefaults, + mainAccountCreds, + ); + } + + private constructor( + private readonly accountCredentials: Map, + private readonly accountDefaults: AwsIntegrationDefaultAccountConfig, + private readonly mainAccountCredentials: AwsCredentials, + ) {} + + /** + * Returns {@link AwsCredentials} for a given AWS account. + * + * @example + * ```ts + * const { provider } = await getCredentials({ + * accountId: '0123456789012', + * }) + * + * const { provider } = await getCredentials({ + * arn: 'arn:aws:ecs:us-west-2:123456789012:service/my-http-service' + * }) + * ``` + * + * @param opts - the AWS account ID or AWS resource ARN + * @returns A promise of {@link AwsCredentials}. + */ + async getCredentials( + opts?: AwsCredentialsProviderOptions, + ): Promise { + // If no options provided, fall back to the main account + if (!opts) { + await fillInAccountId(this.mainAccountCredentials); + return this.mainAccountCredentials; + } + + // Determine the account ID: either explicitly provided or extracted from the provided ARN + let accountId = opts.accountId; + if (opts.arn && !accountId) { + const arnComponents = parse(opts.arn); + accountId = arnComponents.accountId; + } + + // If the account ID was not provided (explicitly or in the ARN), + // fall back to the main account + if (!accountId) { + await fillInAccountId(this.mainAccountCredentials); + return this.mainAccountCredentials; + } + + // Return a cached provider if available + if (this.accountCredentials.has(accountId)) { + return this.accountCredentials.get(accountId)!; + } + + // First, fall back to using the account defaults + if (this.accountDefaults.roleName) { + const config: AwsIntegrationAccountConfig = { + accountId, + roleName: this.accountDefaults.roleName, + partition: this.accountDefaults.partition, + region: this.accountDefaults.region, + externalId: this.accountDefaults.externalId, + }; + const provider = getAccountCredentialsProvider( + config, + this.mainAccountCredentials.provider, + ); + const creds: AwsCredentials = { accountId, provider }; + this.accountCredentials.set(accountId, creds); + return creds; + } + + // Then, fall back to using the main account, but only + // if the account requested matches the main account ID + await fillInAccountId(this.mainAccountCredentials); + if (accountId === this.mainAccountCredentials.accountId) { + return this.mainAccountCredentials; + } + + // Otherwise, the account needs to be explicitly configured in Backstage + throw new Error( + `There is no AWS integration that matches ${accountId}. Please add a configuration for this AWS account.`, + ); + } +} diff --git a/packages/integration-aws-node/src/config.test.ts b/packages/integration-aws-node/src/config.test.ts new file mode 100644 index 0000000000..f761a8ac14 --- /dev/null +++ b/packages/integration-aws-node/src/config.test.ts @@ -0,0 +1,335 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config, ConfigReader } from '@backstage/config'; +import { AwsIntegrationConfig, readAwsIntegrationConfig } from './config'; + +describe('readAwsIntegrationConfig', () => { + function buildConfig(data: Partial): Config { + return new ConfigReader(data); + } + + it('reads all values', () => { + const output = readAwsIntegrationConfig( + buildConfig({ + accounts: [ + { + accountId: '111111111111', + accessKeyId: 'ABC', + secretAccessKey: 'EDF', + roleName: 'hello', + partition: 'aws', + region: 'us-east-1', + externalId: 'world', + }, + { + accountId: '222222222222', + accessKeyId: 'GHI', + secretAccessKey: 'JKL', + }, + { + accountId: '333333333333', + roleName: 'hi', + partition: 'aws-other', + region: 'not-us-east-1', + externalId: 'there', + }, + { + accountId: '444444444444', + profile: 'my-profile', + }, + ], + accountDefaults: { + roleName: 'backstage-role', + partition: 'aws', + region: 'us-east-1', + externalId: 'my-id', + }, + mainAccount: { + accessKeyId: 'GHI', + secretAccessKey: 'JKL', + region: 'ap-northeast-1', + }, + }), + ); + expect(output).toEqual({ + accounts: [ + { + accountId: '111111111111', + accessKeyId: 'ABC', + secretAccessKey: 'EDF', + roleName: 'hello', + partition: 'aws', + region: 'us-east-1', + externalId: 'world', + }, + { + accountId: '222222222222', + accessKeyId: 'GHI', + secretAccessKey: 'JKL', + }, + { + accountId: '333333333333', + roleName: 'hi', + partition: 'aws-other', + region: 'not-us-east-1', + externalId: 'there', + }, + { + accountId: '444444444444', + profile: 'my-profile', + }, + ], + accountDefaults: { + roleName: 'backstage-role', + partition: 'aws', + region: 'us-east-1', + externalId: 'my-id', + }, + mainAccount: { + accessKeyId: 'GHI', + secretAccessKey: 'JKL', + region: 'ap-northeast-1', + }, + }); + }); + + it('reads profile for main account', () => { + const output = readAwsIntegrationConfig( + buildConfig({ + accounts: [ + { + accountId: '111111111111', + accessKeyId: 'ABC', + secretAccessKey: 'EDF', + roleName: 'hello', + partition: 'aws', + region: 'us-east-1', + externalId: 'world', + }, + ], + accountDefaults: { + roleName: 'backstage-role', + partition: 'aws', + region: 'us-east-1', + externalId: 'my-id', + }, + mainAccount: { + profile: 'my-profile', + }, + }), + ); + expect(output).toEqual({ + accounts: [ + { + accountId: '111111111111', + accessKeyId: 'ABC', + secretAccessKey: 'EDF', + roleName: 'hello', + partition: 'aws', + region: 'us-east-1', + externalId: 'world', + }, + ], + accountDefaults: { + roleName: 'backstage-role', + partition: 'aws', + region: 'us-east-1', + externalId: 'my-id', + }, + mainAccount: { + profile: 'my-profile', + }, + }); + }); + + it('does not fail when config is not set', () => { + const output = readAwsIntegrationConfig(buildConfig({})); + expect(output).toEqual({ + accountDefaults: {}, + accounts: [], + mainAccount: {}, + }); + }); + + it('rejects invalid combinations of account attributes', () => { + const validAccount: any = { + accountId: '111111111111', + accessKeyId: 'ABC', + secretAccessKey: 'EDF', + roleName: 'hello', + partition: 'aws', + region: 'us-east-1', + externalId: 'world', + }; + expect(() => + readAwsIntegrationConfig( + buildConfig({ + accounts: [ + validAccount, + { + accountId: '222222222222', + accessKeyId: 'ABC', + }, + ], + }), + ), + ).toThrow(/no secret access key/); + expect(() => + readAwsIntegrationConfig( + buildConfig({ + accounts: [ + validAccount, + { + accountId: '222222222222', + secretAccessKey: 'ABC', + }, + ], + }), + ), + ).toThrow(/no access key ID/); + expect(() => + readAwsIntegrationConfig( + buildConfig({ + accounts: [ + validAccount, + { + accountId: '222222222222', + accessKeyId: 'ABC', + secretAccessKey: 'DEF', + profile: 'my-profile', + }, + ], + }), + ), + ).toThrow(/only one must be specified/); + expect(() => + readAwsIntegrationConfig( + buildConfig({ + accounts: [ + validAccount, + { + accountId: '222222222222', + roleName: 'my-role', + profile: 'my-profile', + }, + ], + }), + ), + ).toThrow(/only one must be specified/); + expect(() => + readAwsIntegrationConfig( + buildConfig({ + accounts: [ + validAccount, + { + accountId: '222222222222', + partition: 'aws', + }, + ], + }), + ), + ).toThrow(/no role name/); + expect(() => + readAwsIntegrationConfig( + buildConfig({ + accounts: [ + validAccount, + { + accountId: '222222222222', + region: 'not-us-east-1', + }, + ], + }), + ), + ).toThrow(/no role name/); + expect(() => + readAwsIntegrationConfig( + buildConfig({ + accounts: [ + validAccount, + { + accountId: '222222222222', + externalId: 'hello', + }, + ], + }), + ), + ).toThrow(/no role name/); + }); + + it('rejects invalid combinations of main account attributes', () => { + expect(() => + readAwsIntegrationConfig( + buildConfig({ + mainAccount: { + accessKeyId: 'ABC', + }, + }), + ), + ).toThrow(/no secret access key/); + expect(() => + readAwsIntegrationConfig( + buildConfig({ + mainAccount: { + secretAccessKey: 'ABC', + }, + }), + ), + ).toThrow(/no access key ID/); + expect(() => + readAwsIntegrationConfig( + buildConfig({ + mainAccount: { + accessKeyId: 'ABC', + secretAccessKey: 'DEF', + profile: 'my-profile', + }, + }), + ), + ).toThrow(/only one must be specified/); + }); + + it('rejects invalid combinations of account default attributes', () => { + expect(() => + readAwsIntegrationConfig( + buildConfig({ + accountDefaults: { + partition: 'aws', + }, + }), + ), + ).toThrow(/no role name/); + expect(() => + readAwsIntegrationConfig( + buildConfig({ + accountDefaults: { + region: 'not-us-east-1', + }, + }), + ), + ).toThrow(/no role name/); + expect(() => + readAwsIntegrationConfig( + buildConfig({ + accountDefaults: { + externalId: 'hello', + }, + }), + ), + ).toThrow(/no role name/); + }); +}); diff --git a/packages/integration-aws-node/src/config.ts b/packages/integration-aws-node/src/config.ts new file mode 100644 index 0000000000..0bc8c0ed06 --- /dev/null +++ b/packages/integration-aws-node/src/config.ts @@ -0,0 +1,307 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +/** + * The configuration parameters for a single AWS account for the AWS integration. + * + * @public + */ +export type AwsIntegrationAccountConfig = { + /** + * The account ID of the target account that this matches on, e.g. "123456789012" + */ + accountId: string; + + /** + * The access key ID for a set of static AWS credentials + */ + accessKeyId?: string; + + /** + * The secret access key for a set of static AWS credentials + */ + secretAccessKey?: string; + + /** + * The configuration profile from a credentials file at ~/.aws/credentials and + * a configuration file at ~/.aws/config. + */ + profile?: string; + + /** + * The IAM role to assume to retrieve temporary AWS credentials + */ + roleName?: string; + + /** + * The AWS partition of the IAM role, e.g. "aws", "aws-cn" + */ + partition?: string; + + /** + * The STS regional endpoint to use when retrieving temporary AWS credentials, e.g. "ap-northeast-1" + */ + region?: string; + + /** + * The unique identifier needed to assume the role to retrieve temporary AWS credentials + */ + externalId?: string; +}; + +/** + * The configuration parameters for the main AWS account for the AWS integration. + * + * @public + */ +export type AwsIntegrationMainAccountConfig = { + /** + * The access key ID for a set of static AWS credentials + */ + accessKeyId?: string; + + /** + * The secret access key for a set of static AWS credentials + */ + secretAccessKey?: string; + + /** + * The configuration profile from a credentials file at ~/.aws/credentials and + * a configuration file at ~/.aws/config. + */ + profile?: string; + + /** + * The STS regional endpoint to use for the main account, e.g. "ap-northeast-1" + */ + region?: string; +}; + +/** + * The default configuration parameters to use for accounts for the AWS integration. + * + * @public + */ +export type AwsIntegrationDefaultAccountConfig = { + /** + * The IAM role to assume to retrieve temporary AWS credentials + */ + roleName?: string; + + /** + * The AWS partition of the IAM role, e.g. "aws", "aws-cn" + */ + partition?: string; + + /** + * The STS regional endpoint to use when retrieving temporary AWS credentials, e.g. "ap-northeast-1" + */ + region?: string; + + /** + * The unique identifier needed to assume the role to retrieve temporary AWS credentials + */ + externalId?: string; +}; + +/** + * The configuration parameters for AWS account integration. + * + * @public + */ +export type AwsIntegrationConfig = { + /** + * Configuration for retrieving AWS accounts credentials + */ + accounts: AwsIntegrationAccountConfig[]; + + /** + * Defaults for retrieving AWS account credentials + */ + accountDefaults: AwsIntegrationDefaultAccountConfig; + + /** + * Main account to use for retrieving AWS account credentials + */ + mainAccount: AwsIntegrationMainAccountConfig; +}; + +/** + * Reads an AWS integration account config. + * + * @param config - The config object of a single account + */ +function readAwsIntegrationAccountConfig( + config: Config, +): AwsIntegrationAccountConfig { + const accountConfig = { + accountId: config.getString('accountId'), + accessKeyId: config.getOptionalString('accessKeyId'), + secretAccessKey: config.getOptionalString('secretAccessKey'), + profile: config.getOptionalString('profile'), + roleName: config.getOptionalString('roleName'), + region: config.getOptionalString('region'), + partition: config.getOptionalString('partition'), + externalId: config.getOptionalString('externalId'), + }; + + // Validate that the account config has the right combination of attributes + if (accountConfig.accessKeyId && !accountConfig.secretAccessKey) { + throw new Error( + `AWS integration account ${accountConfig.accountId} has an access key ID configured, but no secret access key.`, + ); + } + + if (!accountConfig.accessKeyId && accountConfig.secretAccessKey) { + throw new Error( + `AWS integration account ${accountConfig.accountId} has a secret access key configured, but no access key ID`, + ); + } + + if (accountConfig.profile && accountConfig.accessKeyId) { + throw new Error( + `AWS integration account ${accountConfig.accountId} has both an access key ID and a profile configured, but only one must be specified`, + ); + } + + if (accountConfig.profile && accountConfig.roleName) { + throw new Error( + `AWS integration account ${accountConfig.accountId} has both an access key ID and a role name configured, but only one must be specified`, + ); + } + + if (!accountConfig.roleName && accountConfig.externalId) { + throw new Error( + `AWS integration account ${accountConfig.accountId} has an external ID configured, but no role name.`, + ); + } + + if (!accountConfig.roleName && accountConfig.region) { + throw new Error( + `AWS integration account ${accountConfig.accountId} has an STS region configured, but no role name.`, + ); + } + + if (!accountConfig.roleName && accountConfig.partition) { + throw new Error( + `AWS integration account ${accountConfig.accountId} has an IAM partition configured, but no role name.`, + ); + } + + return accountConfig; +} + +/** + * Reads the main AWS integration account config. + * + * @param config - The config object of the main account + */ +function readMainAwsIntegrationAccountConfig( + config: Config, +): AwsIntegrationMainAccountConfig { + const mainAccountConfig = { + accessKeyId: config.getOptionalString('accessKeyId'), + secretAccessKey: config.getOptionalString('secretAccessKey'), + profile: config.getOptionalString('profile'), + region: config.getOptionalString('region'), + }; + + // Validate that the account config has the right combination of attributes + if (mainAccountConfig.accessKeyId && !mainAccountConfig.secretAccessKey) { + throw new Error( + `The main AWS integration account has an access key ID configured, but no secret access key.`, + ); + } + + if (!mainAccountConfig.accessKeyId && mainAccountConfig.secretAccessKey) { + throw new Error( + `The main AWS integration account has a secret access key configured, but no access key ID`, + ); + } + + if (mainAccountConfig.profile && mainAccountConfig.accessKeyId) { + throw new Error( + `The main AWS integration account has both an access key ID and a profile configured, but only one must be specified`, + ); + } + + return mainAccountConfig; +} + +/** + * Reads the default settings for retrieving credentials from AWS integration accounts. + * + * @param config - The config object of the default account settings + */ +function readAwsIntegrationAccountDefaultsConfig( + config: Config, +): AwsIntegrationDefaultAccountConfig { + const defaultAccountConfig = { + roleName: config.getOptionalString('roleName'), + partition: config.getOptionalString('partition'), + region: config.getOptionalString('region'), + externalId: config.getOptionalString('externalId'), + }; + + // Validate that the account config has the right combination of attributes + if (!defaultAccountConfig.roleName && defaultAccountConfig.externalId) { + throw new Error( + `AWS integration account default configuration has an external ID configured, but no role name.`, + ); + } + + if (!defaultAccountConfig.roleName && defaultAccountConfig.region) { + throw new Error( + `AWS integration account default configuration has an STS region configured, but no role name.`, + ); + } + + if (!defaultAccountConfig.roleName && defaultAccountConfig.partition) { + throw new Error( + `AWS integration account default configuration has an IAM partition configured, but no role name.`, + ); + } + + return defaultAccountConfig; +} + +/** + * Reads an AWS integration configuration + * + * @param config - the integration config object + * @public + */ +export function readAwsIntegrationConfig(config: Config): AwsIntegrationConfig { + const accounts = config + .getOptionalConfigArray('accounts') + ?.map(readAwsIntegrationAccountConfig); + const mainAccount = config.has('mainAccount') + ? readMainAwsIntegrationAccountConfig(config.getConfig('mainAccount')) + : {}; + const accountDefaults = config.has('accountDefaults') + ? readAwsIntegrationAccountDefaultsConfig( + config.getConfig('accountDefaults'), + ) + : {}; + + return { + accounts: accounts ?? [], + mainAccount, + accountDefaults, + }; +} diff --git a/packages/integration-aws-node/src/index.ts b/packages/integration-aws-node/src/index.ts new file mode 100644 index 0000000000..d0a86acb47 --- /dev/null +++ b/packages/integration-aws-node/src/index.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { readAwsIntegrationConfig } from './config'; +export type { + AwsIntegrationConfig, + AwsIntegrationAccountConfig, + AwsIntegrationDefaultAccountConfig, + AwsIntegrationMainAccountConfig, +} from './config'; +export { DefaultAwsCredentialsProvider } from './DefaultAwsCredentialsProvider'; +export type { + AwsCredentials, + AwsCredentialsProvider, + AwsCredentialsProviderOptions, +} from './types'; diff --git a/packages/integration-aws-node/src/types.ts b/packages/integration-aws-node/src/types.ts new file mode 100644 index 0000000000..44f7850cf8 --- /dev/null +++ b/packages/integration-aws-node/src/types.ts @@ -0,0 +1,57 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { AwsCredentialIdentityProvider } from '@aws-sdk/types'; + +/** + * A set of credentials information for an AWS account. + * + * @public + */ +export type AwsCredentials = { + accountId?: string; + stsRegion?: string; + provider: AwsCredentialIdentityProvider; +}; + +/** + * The options for specifying the AWS credentials to retrieve. + * + * @public + */ +export type AwsCredentialsProviderOptions = { + /** + * The AWS account ID, e.g. '0123456789012' + */ + accountId?: string; + + /** + * The resource ARN that will be accessed with the returned credentials. + * If account ID or region are not specified, they will be inferred from the ARN. + */ + arn?: string; +}; + +/** + * This allows implementations to be provided to retrieve AWS credentials. + * + * @public + */ +export interface AwsCredentialsProvider { + /** + * Get credentials for an AWS account. + */ + getCredentials(opts?: AwsCredentialsProviderOptions): Promise; +} diff --git a/yarn.lock b/yarn.lock index 98614b0736..efdb297cbf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -639,7 +639,7 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/client-sts@npm:3.218.0": +"@aws-sdk/client-sts@npm:3.218.0, @aws-sdk/client-sts@npm:^3.208.0": version: 3.218.0 resolution: "@aws-sdk/client-sts@npm:3.218.0" dependencies: @@ -748,7 +748,7 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/credential-provider-node@npm:3.218.0": +"@aws-sdk/credential-provider-node@npm:3.218.0, @aws-sdk/credential-provider-node@npm:^3.208.0": version: 3.218.0 resolution: "@aws-sdk/credential-provider-node@npm:3.218.0" dependencies: @@ -1386,7 +1386,7 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/util-arn-parser@npm:3.208.0": +"@aws-sdk/util-arn-parser@npm:3.208.0, @aws-sdk/util-arn-parser@npm:^3.208.0": version: 3.208.0 resolution: "@aws-sdk/util-arn-parser@npm:3.208.0" dependencies: @@ -4135,6 +4135,25 @@ __metadata: languageName: unknown linkType: soft +"@backstage/integration-aws-node@workspace:packages/integration-aws-node": + version: 0.0.0-use.local + resolution: "@backstage/integration-aws-node@workspace:packages/integration-aws-node" + dependencies: + "@aws-sdk/client-sts": ^3.208.0 + "@aws-sdk/credential-provider-node": ^3.208.0 + "@aws-sdk/credential-providers": ^3.208.0 + "@aws-sdk/types": ^3.208.0 + "@aws-sdk/util-arn-parser": ^3.208.0 + "@backstage/cli": "workspace:^" + "@backstage/config": "workspace:^" + "@backstage/config-loader": "workspace:^" + "@backstage/errors": "workspace:^" + "@backstage/test-utils": "workspace:^" + aws-sdk-client-mock: ^2.0.0 + aws-sdk-client-mock-jest: ^2.0.0 + languageName: unknown + linkType: soft + "@backstage/integration-react@npm:^1.1.5": version: 1.1.6 resolution: "@backstage/integration-react@npm:1.1.6" @@ -10137,6 +10156,15 @@ __metadata: languageName: node linkType: hard +"@jest/expect-utils@npm:^28.1.3": + version: 28.1.3 + resolution: "@jest/expect-utils@npm:28.1.3" + dependencies: + jest-get-type: ^28.0.2 + checksum: 808ea3a68292a7e0b95490fdd55605c430b4cf209ea76b5b61bfb2a1badcb41bc046810fe4e364bd5fe04663978aa2bd73d8f8465a761dd7c655aeb44cf22987 + languageName: node + linkType: hard + "@jest/expect-utils@npm:^29.3.1": version: 29.3.1 resolution: "@jest/expect-utils@npm:29.3.1" @@ -10219,6 +10247,15 @@ __metadata: languageName: node linkType: hard +"@jest/schemas@npm:^28.1.3": + version: 28.1.3 + resolution: "@jest/schemas@npm:28.1.3" + dependencies: + "@sinclair/typebox": ^0.24.1 + checksum: 3cf1d4b66c9c4ffda58b246de1ddcba8e6ad085af63dccdf07922511f13b68c0cc480a7bc620cb4f3099a6f134801c747e1df7bfc7a4ef4dceefbdea3e31e1de + languageName: node + linkType: hard + "@jest/schemas@npm:^29.0.0": version: 29.0.0 resolution: "@jest/schemas@npm:29.0.0" @@ -10299,6 +10336,20 @@ __metadata: languageName: node linkType: hard +"@jest/types@npm:^28.1.3": + version: 28.1.3 + resolution: "@jest/types@npm:28.1.3" + dependencies: + "@jest/schemas": ^28.1.3 + "@types/istanbul-lib-coverage": ^2.0.0 + "@types/istanbul-reports": ^3.0.0 + "@types/node": "*" + "@types/yargs": ^17.0.8 + chalk: ^4.0.0 + checksum: 1e258d9c063fcf59ebc91e46d5ea5984674ac7ae6cae3e50aa780d22b4405bf2c925f40350bf30013839eb5d4b5e521d956ddf8f3b7c78debef0e75a07f57350 + languageName: node + linkType: hard + "@jest/types@npm:^29.3.1": version: 29.3.1 resolution: "@jest/types@npm:29.3.1" @@ -14168,6 +14219,16 @@ __metadata: languageName: node linkType: hard +"@types/jest@npm:^28.1.3": + version: 28.1.8 + resolution: "@types/jest@npm:28.1.8" + dependencies: + expect: ^28.0.0 + pretty-format: ^28.0.0 + checksum: d4cd36158a3ae1d4b42cc48a77c95de74bc56b84cf81e09af3ee0399c34f4a7da8ab9e787570f10004bd642f9e781b0033c37327fbbf4a8e4b6e37e8ee3693a7 + languageName: node + linkType: hard + "@types/jquery@npm:^3.3.34": version: 3.5.14 resolution: "@types/jquery@npm:3.5.14" @@ -16664,6 +16725,18 @@ __metadata: languageName: node linkType: hard +"aws-sdk-client-mock-jest@npm:^2.0.0": + version: 2.0.0 + resolution: "aws-sdk-client-mock-jest@npm:2.0.0" + dependencies: + "@types/jest": ^28.1.3 + tslib: ^2.1.0 + peerDependencies: + aws-sdk-client-mock: 2.0.0 + checksum: 57dc95b52f0c41166af44c743f298992f688ad55c4492fd2f09d7be59277c57a9c0ce408c8081b88358cb75c802922ca9d299e747797b023518ea57c01fa4ece + languageName: node + linkType: hard + "aws-sdk-client-mock@npm:^2.0.0": version: 2.0.1 resolution: "aws-sdk-client-mock@npm:2.0.1" @@ -20223,6 +20296,13 @@ __metadata: languageName: node linkType: hard +"diff-sequences@npm:^28.1.1": + version: 28.1.1 + resolution: "diff-sequences@npm:28.1.1" + checksum: e2529036505567c7ca5a2dea86b6bcd1ca0e3ae63bf8ebf529b8a99cfa915bbf194b7021dc1c57361a4017a6d95578d4ceb29fabc3232a4f4cb866a2726c7690 + languageName: node + linkType: hard + "diff-sequences@npm:^29.3.1": version: 29.3.1 resolution: "diff-sequences@npm:29.3.1" @@ -22015,6 +22095,19 @@ __metadata: languageName: node linkType: hard +"expect@npm:^28.0.0": + version: 28.1.3 + resolution: "expect@npm:28.1.3" + dependencies: + "@jest/expect-utils": ^28.1.3 + jest-get-type: ^28.0.2 + jest-matcher-utils: ^28.1.3 + jest-message-util: ^28.1.3 + jest-util: ^28.1.3 + checksum: 101e0090de300bcafedb7dbfd19223368a2251ce5fe0105bbb6de5720100b89fb6b64290ebfb42febc048324c76d6a4979cdc4b61eb77747857daf7a5de9b03d + languageName: node + linkType: hard + "expect@npm:^29.0.0, expect@npm:^29.3.1": version: 29.3.1 resolution: "expect@npm:29.3.1" @@ -25656,6 +25749,18 @@ __metadata: languageName: node linkType: hard +"jest-diff@npm:^28.1.3": + version: 28.1.3 + resolution: "jest-diff@npm:28.1.3" + dependencies: + chalk: ^4.0.0 + diff-sequences: ^28.1.1 + jest-get-type: ^28.0.2 + pretty-format: ^28.1.3 + checksum: fa8583e0ccbe775714ce850b009be1b0f6b17a4b6759f33ff47adef27942ebc610dbbcc8a5f7cfb7f12b3b3b05afc9fb41d5f766674616025032ff1e4f9866e0 + languageName: node + linkType: hard + "jest-diff@npm:^29.3.1": version: 29.3.1 resolution: "jest-diff@npm:29.3.1" @@ -25725,6 +25830,13 @@ __metadata: languageName: node linkType: hard +"jest-get-type@npm:^28.0.2": + version: 28.0.2 + resolution: "jest-get-type@npm:28.0.2" + checksum: 5281d7c89bc8156605f6d15784f45074f4548501195c26e9b188742768f72d40948252d13230ea905b5349038865a1a8eeff0e614cc530ff289dfc41fe843abd + languageName: node + linkType: hard + "jest-get-type@npm:^29.2.0": version: 29.2.0 resolution: "jest-get-type@npm:29.2.0" @@ -25765,6 +25877,18 @@ __metadata: languageName: node linkType: hard +"jest-matcher-utils@npm:^28.1.3": + version: 28.1.3 + resolution: "jest-matcher-utils@npm:28.1.3" + dependencies: + chalk: ^4.0.0 + jest-diff: ^28.1.3 + jest-get-type: ^28.0.2 + pretty-format: ^28.1.3 + checksum: 6b34f0cf66f6781e92e3bec97bf27796bd2ba31121e5c5997218d9adba6deea38a30df5203937d6785b68023ed95cbad73663cc9aad6fb0cb59aeb5813a58daf + languageName: node + linkType: hard + "jest-matcher-utils@npm:^29.3.1": version: 29.3.1 resolution: "jest-matcher-utils@npm:29.3.1" @@ -25777,6 +25901,23 @@ __metadata: languageName: node linkType: hard +"jest-message-util@npm:^28.1.3": + version: 28.1.3 + resolution: "jest-message-util@npm:28.1.3" + dependencies: + "@babel/code-frame": ^7.12.13 + "@jest/types": ^28.1.3 + "@types/stack-utils": ^2.0.0 + chalk: ^4.0.0 + graceful-fs: ^4.2.9 + micromatch: ^4.0.4 + pretty-format: ^28.1.3 + slash: ^3.0.0 + stack-utils: ^2.0.3 + checksum: 1f266854166dcc6900d75a88b54a25225a2f3710d463063ff1c99021569045c35c7d58557b25447a17eb3a65ce763b2f9b25550248b468a9d4657db365f39e96 + languageName: node + linkType: hard + "jest-message-util@npm:^29.3.1": version: 29.3.1 resolution: "jest-message-util@npm:29.3.1" @@ -25942,6 +26083,20 @@ __metadata: languageName: node linkType: hard +"jest-util@npm:^28.1.3": + version: 28.1.3 + resolution: "jest-util@npm:28.1.3" + dependencies: + "@jest/types": ^28.1.3 + "@types/node": "*" + chalk: ^4.0.0 + ci-info: ^3.2.0 + graceful-fs: ^4.2.9 + picomatch: ^2.2.3 + checksum: fd6459742c941f070223f25e38a2ac0719aad92561591e9fb2a50d602a5d19d754750b79b4074327a42b00055662b95da3b006542ceb8b54309da44d4a62e721 + languageName: node + linkType: hard + "jest-util@npm:^29.3.1": version: 29.3.1 resolution: "jest-util@npm:29.3.1" @@ -31752,6 +31907,18 @@ __metadata: languageName: node linkType: hard +"pretty-format@npm:^28.0.0, pretty-format@npm:^28.1.3": + version: 28.1.3 + resolution: "pretty-format@npm:28.1.3" + dependencies: + "@jest/schemas": ^28.1.3 + ansi-regex: ^5.0.1 + ansi-styles: ^5.0.0 + react-is: ^18.0.0 + checksum: e69f857358a3e03d271252d7524bec758c35e44680287f36c1cb905187fbc82da9981a6eb07edfd8a03bc3cbeebfa6f5234c13a3d5b59f2bbdf9b4c4053e0a7f + languageName: node + linkType: hard + "pretty-format@npm:^29.0.0, pretty-format@npm:^29.3.1": version: 29.3.1 resolution: "pretty-format@npm:29.3.1" From e40790d0c2e50c36ddd0cd0fa28d6011ab61ccff Mon Sep 17 00:00:00 2001 From: Clare Liguori Date: Mon, 28 Nov 2022 13:23:26 -0800 Subject: [PATCH 040/437] Use integration-aws-node for credentials in S3 Techdocs Signed-off-by: Clare Liguori --- .changeset/silly-knives-warn.md | 5 + docs/features/techdocs/configuration.md | 16 +- docs/features/techdocs/using-cloud-storage.md | 9 +- plugins/techdocs-backend/config.d.ts | 16 +- plugins/techdocs-node/package.json | 1 + .../src/stages/publish/awsS3.test.ts | 147 ++++++++++++++---- .../techdocs-node/src/stages/publish/awsS3.ts | 60 ++++--- .../src/stages/publish/publish.ts | 2 +- yarn.lock | 3 +- 9 files changed, 198 insertions(+), 61 deletions(-) create mode 100644 .changeset/silly-knives-warn.md diff --git a/.changeset/silly-knives-warn.md b/.changeset/silly-knives-warn.md new file mode 100644 index 0000000000..f5fb61b493 --- /dev/null +++ b/.changeset/silly-knives-warn.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs-node': patch +--- + +Add support for specifying an S3 bucket's account ID and retrieving the credentials from the `aws` app config section. This is now the preferred way to configure AWS credentials for Techdocs. diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 4e806296e5..097d31f760 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -106,8 +106,20 @@ techdocs: # If not set, the default location will be the root of the storage bucket bucketRootPath: '/' - # (Optional) An API key is required to write to a storage bucket. - # If not set, environment variables or aws config file will be used to authenticate. + # (Optional) The AWS account ID where the storage bucket is located. + # Credentials for the account ID must be configured in the 'aws' app config section. + # See the integration-aws-node package for details on how to configure credentials in + # the 'aws' app config section. + # https://www.npmjs.com/package/@backstage/integration-aws-node + # If account ID is not set and no credentials are set, environment variables or aws config file will be used to authenticate. + # https://www.npmjs.com/package/@aws-sdk/credential-provider-node + # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-credentials-node.html + accountId: ${TECHDOCS_AWSS3_ACCOUNT_ID} + + # (Optional) AWS credentials to use to write to the storage bucket. + # This configuration section is now deprecated. + # Configuring the account ID is now preferred, with credentials in the 'aws' app config section. + # If credentials are not set and no account ID is set, environment variables or aws config file will be used to authenticate. # https://www.npmjs.com/package/@aws-sdk/credential-provider-node # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-credentials-node.html credentials: diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 3ea57edc6b..f38a806413 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -244,10 +244,13 @@ techdocs: type: 'awsS3' awsS3: bucketName: 'name-of-techdocs-storage-bucket' + accountId: '123456789012' region: ${AWS_REGION} - credentials: - accessKeyId: ${AWS_ACCESS_KEY_ID} - secretAccessKey: ${AWS_SECRET_ACCESS_KEY} +aws: + accounts: + - accountId: '123456789012' + accessKeyId: ${AWS_ACCESS_KEY_ID} + secretAccessKey: ${AWS_SECRET_ACCESS_KEY} ``` Refer to the diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts index ec35505a78..3a05beabdc 100644 --- a/plugins/techdocs-backend/config.d.ts +++ b/plugins/techdocs-backend/config.d.ts @@ -81,9 +81,23 @@ export interface Config { * Required when 'type' is set to awsS3 */ awsS3?: { + /** + * (Optional) The AWS account ID where the storage bucket is located. + * Credentials for the account ID will be sourced from the 'aws' app config section. + * See the + * [integration-aws-node package](https://github.com/backstage/backstage/blob/master/packages/integration-aws-node/README.md) + * for details on how to configure the credentials in the app config. + * If account ID is not set and no credentials are set, environment variables or aws config file will be used to authenticate. + * @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html + * @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html + * @visibility secret + */ + accountId?: string; /** * (Optional) Credentials used to access a storage bucket. - * If not set, environment variables or aws config file will be used to authenticate. + * This section is now deprecated. Configuring the account ID is now preferred, with credentials in the 'aws' + * app config section. + * If not set and no account ID is set, environment variables or aws config file will be used to authenticate. * @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html * @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html * @visibility secret diff --git a/plugins/techdocs-node/package.json b/plugins/techdocs-node/package.json index 668fe58239..631de02b55 100644 --- a/plugins/techdocs-node/package.json +++ b/plugins/techdocs-node/package.json @@ -50,6 +50,7 @@ "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", + "@backstage/integration-aws-node": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@google-cloud/storage": "^6.0.0", "@trendyol-js/openstack-swift-sdk": "^0.0.5", diff --git a/plugins/techdocs-node/src/stages/publish/awsS3.test.ts b/plugins/techdocs-node/src/stages/publish/awsS3.test.ts index 08facf2208..ff61ac58a2 100644 --- a/plugins/techdocs-node/src/stages/publish/awsS3.test.ts +++ b/plugins/techdocs-node/src/stages/publish/awsS3.test.ts @@ -27,6 +27,11 @@ import { import { getVoidLogger } from '@backstage/backend-common'; import { Entity, DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; +import { + AwsCredentials, + AwsCredentialsProviderOptions, + DefaultAwsCredentialsProvider, +} from '@backstage/integration-aws-node'; import { mockClient, AwsClientStub } from 'aws-sdk-client-mock'; import express from 'express'; import request from 'supertest'; @@ -40,6 +45,21 @@ import { Readable } from 'stream'; const env = process.env; let s3Mock: AwsClientStub; +function getMockCredentials(): Promise { + return Promise.resolve({ + provider: async () => { + return Promise.resolve({ + accessKeyId: 'MY_ACCESS_KEY_ID', + secretAccessKey: 'MY_SECRET_ACCESS_KEY', + }); + }, + }); +} +const credsProviderMock = jest.spyOn( + DefaultAwsCredentialsProvider.prototype, + 'getCredentials', +); + const getEntityRootDir = (entity: Entity) => { const { kind, @@ -70,7 +90,7 @@ const logger = getVoidLogger(); const loggerInfoSpy = jest.spyOn(logger, 'info'); const loggerErrorSpy = jest.spyOn(logger, 'error'); -const createPublisherFromConfig = ({ +const createPublisherFromConfig = async ({ bucketName = 'bucketName', bucketRootPath = '/', legacyUseCaseSensitiveTripletPaths = false, @@ -86,10 +106,7 @@ const createPublisherFromConfig = ({ publisher: { type: 'awsS3', awsS3: { - credentials: { - accessKeyId: 'accessKeyId', - secretAccessKey: 'secretAccessKey', - }, + accountId: '111111111111', bucketName, bucketRootPath, sse, @@ -97,9 +114,18 @@ const createPublisherFromConfig = ({ }, legacyUseCaseSensitiveTripletPaths, }, + aws: { + accounts: [ + { + accountId: '111111111111', + accessKeyId: 'my-access-key', + secretAccessKey: 'my-secret-access-key', + }, + ], + }, }); - return AwsS3Publish.fromConfig(mockConfig, logger); + return await AwsS3Publish.fromConfig(mockConfig, logger); }; describe('AwsS3Publish', () => { @@ -151,6 +177,11 @@ describe('AwsS3Publish', () => { process.env = { ...env }; process.env.AWS_REGION = 'us-west-2'; + jest.resetAllMocks(); + credsProviderMock.mockImplementation((_?: AwsCredentialsProviderOptions) => + getMockCredentials(), + ); + mockFs({ [directory]: files, }); @@ -215,16 +246,64 @@ describe('AwsS3Publish', () => { process.env = env; }); + describe('buildCredentials', () => { + it('should retrieve credentials for a specific account ID', async () => { + await createPublisherFromConfig(); + expect(credsProviderMock).toHaveBeenCalledWith({ + accountId: '111111111111', + }); + expect(credsProviderMock).toHaveBeenCalledTimes(1); + }); + + it('should retrieve default credentials when no config is present', async () => { + const mockConfig = new ConfigReader({ + techdocs: { + publisher: { + type: 'awsS3', + awsS3: { + bucketName: 'bucketName', + }, + }, + }, + }); + + await AwsS3Publish.fromConfig(mockConfig, logger); + expect(credsProviderMock).toHaveBeenCalledWith(); + expect(credsProviderMock).toHaveBeenCalledTimes(1); + }); + + it('should fall back to deprecated method of retrieving credentials', async () => { + const mockConfig = new ConfigReader({ + techdocs: { + publisher: { + type: 'awsS3', + awsS3: { + credentials: { + accessKeyId: 'accessKeyId', + secretAccessKey: 'secretAccessKey', + }, + bucketName: 'bucketName', + bucketRootPath: '/', + }, + }, + }, + }); + + await AwsS3Publish.fromConfig(mockConfig, logger); + expect(credsProviderMock).toHaveBeenCalledTimes(0); + }); + }); + describe('getReadiness', () => { it('should validate correct config', async () => { - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); expect(await publisher.getReadiness()).toEqual({ isAvailable: true, }); }); it('should reject incorrect config', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketName: 'errorBucket', }); expect(await publisher.getReadiness()).toEqual({ @@ -235,7 +314,7 @@ describe('AwsS3Publish', () => { describe('publish', () => { it('should publish a directory', async () => { - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); expect(await publisher.publish({ entity, directory })).toMatchObject({ objects: expect.arrayContaining([ 'default/component/backstage/404.html', @@ -246,7 +325,7 @@ describe('AwsS3Publish', () => { }); it('should publish a directory as well when legacy casing is used', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); expect(await publisher.publish({ entity, directory })).toMatchObject({ @@ -259,7 +338,7 @@ describe('AwsS3Publish', () => { }); it('should publish a directory when root path is specified', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: 'backstage-data/techdocs', }); expect(await publisher.publish({ entity, directory })).toMatchObject({ @@ -272,7 +351,7 @@ describe('AwsS3Publish', () => { }); it('should publish a directory when root path is specified and legacy casing is used', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: 'backstage-data/techdocs', legacyUseCaseSensitiveTripletPaths: true, }); @@ -286,7 +365,7 @@ describe('AwsS3Publish', () => { }); it('should publish a directory when sse is specified', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ sse: 'aws:kms', }); expect(await publisher.publish({ entity, directory })).toMatchObject({ @@ -307,7 +386,7 @@ describe('AwsS3Publish', () => { 'generatedDirectory', ); - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); const fails = publisher.publish({ entity, @@ -327,7 +406,9 @@ describe('AwsS3Publish', () => { it('should delete stale files after upload', async () => { const bucketName = 'delete_stale_files_success'; - const publisher = createPublisherFromConfig({ bucketName: bucketName }); + const publisher = await createPublisherFromConfig({ + bucketName: bucketName, + }); await publisher.publish({ entity, directory }); expect(loggerInfoSpy).toHaveBeenLastCalledWith( `Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: 1`, @@ -336,7 +417,9 @@ describe('AwsS3Publish', () => { it('should log error when the stale files deletion fails', async () => { const bucketName = 'delete_stale_files_error'; - const publisher = createPublisherFromConfig({ bucketName: bucketName }); + const publisher = await createPublisherFromConfig({ + bucketName: bucketName, + }); await publisher.publish({ entity, directory }); expect(loggerErrorSpy).toHaveBeenLastCalledWith( 'Unable to delete file(s) from AWS S3. Error: Message', @@ -346,13 +429,13 @@ describe('AwsS3Publish', () => { describe('hasDocsBeenGenerated', () => { it('should return true if docs has been generated', async () => { - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); await publisher.publish({ entity, directory }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); it('should return true if docs has been generated even if the legacy case is enabled', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); await publisher.publish({ entity, directory }); @@ -360,7 +443,7 @@ describe('AwsS3Publish', () => { }); it('should return true if docs has been generated if root path is specified', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: 'backstage-data/techdocs', }); await publisher.publish({ entity, directory }); @@ -368,7 +451,7 @@ describe('AwsS3Publish', () => { }); it('should return true if docs has been generated if root path is specified and legacy casing is used', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: 'backstage-data/techdocs', legacyUseCaseSensitiveTripletPaths: true, }); @@ -377,7 +460,7 @@ describe('AwsS3Publish', () => { }); it('should return false if docs has not been generated', async () => { - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); expect( await publisher.hasDocsBeenGenerated({ kind: 'entity', @@ -392,7 +475,7 @@ describe('AwsS3Publish', () => { describe('fetchTechDocsMetadata', () => { it('should return tech docs metadata', async () => { - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( techdocsMetadata, @@ -400,7 +483,7 @@ describe('AwsS3Publish', () => { }); it('should return tech docs metadata even if the legacy case is enabled', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); await publisher.publish({ entity, directory }); @@ -410,7 +493,7 @@ describe('AwsS3Publish', () => { }); it('should return tech docs metadata even if root path is specified', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: 'backstage-data/techdocs', }); await publisher.publish({ entity, directory }); @@ -420,7 +503,7 @@ describe('AwsS3Publish', () => { }); it('should return tech docs metadata if root path is specified and legacy casing is used', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: 'backstage-data/techdocs', legacyUseCaseSensitiveTripletPaths: true, }); @@ -442,7 +525,7 @@ describe('AwsS3Publish', () => { techdocsMetadataContent.replace(/"/g, "'"), ); - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( @@ -453,7 +536,7 @@ describe('AwsS3Publish', () => { }); it('should return an error if the techdocs_metadata.json file is not present', async () => { - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); const invalidEntityName = { namespace: 'invalid', @@ -477,7 +560,7 @@ describe('AwsS3Publish', () => { }; }); - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); const invalidEntityName = { namespace: 'invalid', @@ -501,7 +584,7 @@ describe('AwsS3Publish', () => { let app: express.Express; beforeEach(async () => { - const publisher = createPublisherFromConfig(); + const publisher = await createPublisherFromConfig(); await publisher.publish({ entity, directory }); app = express().use(publisher.docsRouter()); }); @@ -521,7 +604,7 @@ describe('AwsS3Publish', () => { }); it('should pass expected object path to bucket even if the legacy case is enabled', async () => { - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); await publisher.publish({ entity, directory }); @@ -541,7 +624,7 @@ describe('AwsS3Publish', () => { it('should pass expected object path to bucket if root path is specified', async () => { const rootPath = 'backstage-data/techdocs'; - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: rootPath, }); await publisher.publish({ entity, directory }); @@ -561,7 +644,7 @@ describe('AwsS3Publish', () => { it('should pass expected object path to bucket if root path is specified and legacy case is enabled', async () => { const rootPath = 'backstage-data/techdocs'; - const publisher = createPublisherFromConfig({ + const publisher = await createPublisherFromConfig({ bucketRootPath: rootPath, legacyUseCaseSensitiveTripletPaths: true, }); diff --git a/plugins/techdocs-node/src/stages/publish/awsS3.ts b/plugins/techdocs-node/src/stages/publish/awsS3.ts index 7c2405cc1c..c71a25cfe4 100644 --- a/plugins/techdocs-node/src/stages/publish/awsS3.ts +++ b/plugins/techdocs-node/src/stages/publish/awsS3.ts @@ -16,6 +16,10 @@ import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError, ForwardedError } from '@backstage/errors'; +import { + AwsCredentialsProvider, + DefaultAwsCredentialsProvider, +} from '@backstage/integration-aws-node'; import { GetObjectCommand, CopyObjectCommand, @@ -27,12 +31,9 @@ import { ListObjectsV2Command, S3Client, } from '@aws-sdk/client-s3'; -import { - fromNodeProviderChain, - fromTemporaryCredentials, -} from '@aws-sdk/credential-providers'; +import { fromTemporaryCredentials } from '@aws-sdk/credential-providers'; import { Upload } from '@aws-sdk/lib-storage'; -import { CredentialProvider } from '@aws-sdk/types'; +import { AwsCredentialIdentityProvider } from '@aws-sdk/types'; import express from 'express'; import fs from 'fs-extra'; import JSON5 from 'json5'; @@ -97,7 +98,10 @@ export class AwsS3Publish implements PublisherBase { this.sse = options.sse; } - static fromConfig(config: Config, logger: Logger): PublisherBase { + static async fromConfig( + config: Config, + logger: Logger, + ): Promise { let bucketName = ''; try { bucketName = config.getString('techdocs.publisher.awsS3.bucketName'); @@ -121,17 +125,21 @@ export class AwsS3Publish implements PublisherBase { // or AWS shared credentials file at ~/.aws/credentials will be used. const region = config.getOptionalString('techdocs.publisher.awsS3.region'); - // Credentials is an optional config. If missing, the default ways of authenticating AWS SDK V2 will be used. - // 1. AWS environment variables - // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html - // 2. AWS shared credentials file at ~/.aws/credentials - // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-shared.html - // 3. IAM Roles for EC2 - // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-iam.html + // Credentials can optionally be configured by specifying the AWS account ID, which will retrieve credentials + // for the account from the 'aws' section of the app config. + // Credentials can also optionally be directly configured in the techdocs awsS3 config, but this method is + // deprecated. + // If no credentials are configured, the AWS SDK V3's default credential chain will be used. + const accountId = config.getOptionalString( + 'techdocs.publisher.awsS3.accountId', + ); const credentialsConfig = config.getOptionalConfig( 'techdocs.publisher.awsS3.credentials', ); - const credentials = AwsS3Publish.buildCredentials( + const credsProvider = DefaultAwsCredentialsProvider.fromConfig(config); + const credentials = await AwsS3Publish.buildCredentials( + credsProvider, + accountId, credentialsConfig, region, ); @@ -174,7 +182,7 @@ export class AwsS3Publish implements PublisherBase { private static buildStaticCredentials( accessKeyId: string, secretAccessKey: string, - ): CredentialProvider { + ): AwsCredentialIdentityProvider { return async () => { return Promise.resolve({ accessKeyId, @@ -183,20 +191,30 @@ export class AwsS3Publish implements PublisherBase { }; } - private static buildCredentials( + private static async buildCredentials( + credsProvider: AwsCredentialsProvider, + accountId?: string, config?: Config, region?: string, - ): CredentialProvider { - if (!config) { - return fromNodeProviderChain(); + ): Promise { + // Pull credentials for the specified account ID from the 'aws' config section + if (accountId) { + return (await credsProvider.getCredentials({ accountId })).provider; } + // Fall back to the default credential chain if neither account ID + // nor explicit credentials are provided + if (!config) { + return (await credsProvider.getCredentials()).provider; + } + + // Pull credentials from the techdocs config section (deprecated) const accessKeyId = config.getOptionalString('accessKeyId'); const secretAccessKey = config.getOptionalString('secretAccessKey'); - const explicitCredentials: CredentialProvider = + const explicitCredentials: AwsCredentialIdentityProvider = accessKeyId && secretAccessKey ? AwsS3Publish.buildStaticCredentials(accessKeyId, secretAccessKey) - : fromNodeProviderChain(); + : (await credsProvider.getCredentials()).provider; const roleArn = config.getOptionalString('roleArn'); if (roleArn) { diff --git a/plugins/techdocs-node/src/stages/publish/publish.ts b/plugins/techdocs-node/src/stages/publish/publish.ts index bc7f50b5a3..2d9d07eb83 100644 --- a/plugins/techdocs-node/src/stages/publish/publish.ts +++ b/plugins/techdocs-node/src/stages/publish/publish.ts @@ -47,7 +47,7 @@ export class Publisher { return GoogleGCSPublish.fromConfig(config, logger); case 'awsS3': logger.info('Creating AWS S3 Bucket publisher for TechDocs'); - return AwsS3Publish.fromConfig(config, logger); + return await AwsS3Publish.fromConfig(config, logger); case 'azureBlobStorage': logger.info( 'Creating Azure Blob Storage Container publisher for TechDocs', diff --git a/yarn.lock b/yarn.lock index efdb297cbf..4041e30e7b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4135,7 +4135,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/integration-aws-node@workspace:packages/integration-aws-node": +"@backstage/integration-aws-node@workspace:^, @backstage/integration-aws-node@workspace:packages/integration-aws-node": version: 0.0.0-use.local resolution: "@backstage/integration-aws-node@workspace:packages/integration-aws-node" dependencies: @@ -8142,6 +8142,7 @@ __metadata: "@backstage/config": "workspace:^" "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" + "@backstage/integration-aws-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@google-cloud/storage": ^6.0.0 "@trendyol-js/openstack-swift-sdk": ^0.0.5 From b78b725ee7ce9104c19f7756a9babbcc0af6003f Mon Sep 17 00:00:00 2001 From: Jonathan Nagayoshi Date: Tue, 29 Nov 2022 03:04:28 +0000 Subject: [PATCH 041/437] feat: implemented devcontainers support for developing in the backstage repository locally Signed-off-by: Jonathan Nagayoshi --- .devcontainer/Dockerfile | 5 +++++ .devcontainer/devcontainer.json | 30 ++++++++++++++++++++++++++++++ .devcontainer/postCreate.sh | 3 +++ README.md | 1 + 4 files changed, 39 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100755 .devcontainer/postCreate.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..35459b8d97 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,5 @@ +FROM mcr.microsoft.com/devcontainers/typescript-node:18 + +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install chromium \ + && apt-get -y install build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..075e0c2ec9 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,30 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/typescript-node +{ + "name": "Base Backstage Workspace", + "build": { "dockerfile": "Dockerfile" }, + "features": { + "ghcr.io/devcontainers/features/common-utils:1": {}, + "ghcr.io/devcontainers/features/docker-from-docker:1": {}, + "ghcr.io/devcontainers-contrib/features/mkdocs:1": {} + }, + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [3000, 7007], + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "bash .devcontainer/postCreate.sh", + + // Configure tool-specific properties. + "customizations": { + "vscode": { + "extensions": ["Intility.vscode-backstage"] + } + } + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/.devcontainer/postCreate.sh b/.devcontainer/postCreate.sh new file mode 100755 index 0000000000..f0631a4af2 --- /dev/null +++ b/.devcontainer/postCreate.sh @@ -0,0 +1,3 @@ +#!/bin/bash +yarn install +pip install mkdocs-techdocs-core \ No newline at end of file diff --git a/README.md b/README.md index 8a2bd338d9..115f1ed6d6 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ ![Code style](https://img.shields.io/badge/code_style-prettier-ff69b4.svg) [![Codecov](https://img.shields.io/codecov/c/github/backstage/backstage)](https://codecov.io/gh/backstage/backstage) [![](https://img.shields.io/github/v/release/backstage/backstage)](https://github.com/backstage/backstage/releases) +[![VS Code Container](https://img.shields.io/static/v1?label=VS+Code&message=Container&logo=visualstudiocode&color=007ACC&logoColor=007ACC&labelColor=2C2C32)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/backstage/backstage) ## What is Backstage? From bef58bf44210fef30b3a4f2b4cea79ce1176c203 Mon Sep 17 00:00:00 2001 From: Alex Rybchenko Date: Fri, 18 Nov 2022 14:55:29 +0100 Subject: [PATCH 042/437] api report Signed-off-by: Alex Rybchenko --- plugins/scaffolder/api-report.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index f94cb26d21..46b2ea3c8a 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -110,9 +110,9 @@ export type EntityPickerUiOptions = export const EntityTagsPickerFieldExtension: FieldExtensionComponent< string[], { - helperText?: string | undefined; - kinds?: string[] | undefined; showCounts?: boolean | undefined; + kinds?: string[] | undefined; + helperText?: string | undefined; } >; @@ -120,9 +120,9 @@ export const EntityTagsPickerFieldExtension: FieldExtensionComponent< export const EntityTagsPickerFieldSchema: FieldSchema< string[], { - helperText?: string | undefined; - kinds?: string[] | undefined; showCounts?: boolean | undefined; + kinds?: string[] | undefined; + helperText?: string | undefined; } >; From 9000952e872d9ea5b27a5b09afe47722657d87ae Mon Sep 17 00:00:00 2001 From: Alex Rybchenko Date: Tue, 29 Nov 2022 17:55:10 +0100 Subject: [PATCH 043/437] added changeset Signed-off-by: Alex Rybchenko --- .changeset/nasty-dragons-melt.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/nasty-dragons-melt.md diff --git a/.changeset/nasty-dragons-melt.md b/.changeset/nasty-dragons-melt.md new file mode 100644 index 0000000000..b083ebce62 --- /dev/null +++ b/.changeset/nasty-dragons-melt.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': minor +--- + +All form data is now passed to validator functions in 'next' scaffolder, so it's now possible to perform validation for fields that depend on other field values From bfe38c80ea6202fc6233cbb0ee497762148caff1 Mon Sep 17 00:00:00 2001 From: Esther Annorzie Date: Tue, 29 Nov 2022 13:11:27 -0500 Subject: [PATCH 044/437] Add changeset Signed-off-by: Esther Annorzie --- .changeset/many-mangos-behave.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/many-mangos-behave.md diff --git a/.changeset/many-mangos-behave.md b/.changeset/many-mangos-behave.md new file mode 100644 index 0000000000..ff777d3839 --- /dev/null +++ b/.changeset/many-mangos-behave.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-home': minor +--- + +'backstage/home': minor + +If no entities are starred, a call to action message displays. From 0b153881a72a1ed5c6e8c82e22ac1fd3e3f5f0cd Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Wed, 30 Nov 2022 09:42:42 -0600 Subject: [PATCH 045/437] updating terms Signed-off-by: Lucas De Souza --- plugins/catalog-backend/config.d.ts | 11 ++++---- .../src/ingestion/CatalogRules.test.ts | 12 ++++---- .../src/ingestion/CatalogRules.ts | 28 +++++++++++-------- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 40d423d0d4..5da7dd8df9 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -42,11 +42,12 @@ export interface Config { * Limit this rule to a specific location * * Example with a fixed location - * { "type": "url", "target": "https://github.com/a/b/blob/file.yaml} + * { "type": "url", "exact": "https://github.com/a/b/blob/file.yaml"} * * Example using a Regex - * { "type": "url", "match": "https://github.com/a/*} + * { "type": "url", "pattern": "https://github.com/org/*\/blob/master/*.yaml"} * + * Using both exact and pattern will result in an error starting the application */ locations?: Array<{ /** @@ -54,15 +55,15 @@ export interface Config { */ type: string; /** - * The target URL of the location, e.g. + * The exact location, e.g. * "https://github.com/org/repo/blob/master/users.yaml". */ - target?: string; + exact?: string; /** * The pattern allowed for the location, e.g. * "https://github.com/org/*\/blob/master/*.yaml. */ - match?: string; + pattern?: string; }>; }>; diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index 06bb1389a9..34e89fad29 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -54,7 +54,7 @@ const location: Record = { }; describe('DefaultCatalogRulesEnforcer', () => { - it('should throw an error if both match and target are used', () => { + it('should throw an error if both pattern and exact are used', () => { expect(() => DefaultCatalogRulesEnforcer.fromConfig( new ConfigReader({ @@ -65,8 +65,8 @@ describe('DefaultCatalogRulesEnforcer', () => { locations: [ { type: 'url', - match: 'https://github.com/b/**', - target: 'https://github.com/a/b/blob/master/w.yaml', + pattern: 'https://github.com/b/**', + exact: 'https://github.com/a/b/blob/master/w.yaml', }, ], }, @@ -74,7 +74,7 @@ describe('DefaultCatalogRulesEnforcer', () => { }, }), ), - ).toThrow(/cannot have both target and match values/i); + ).toThrow(/cannot have both exact and pattern values/i); }); it('should deny by default', () => { const enforcer = new DefaultCatalogRulesEnforcer([]); @@ -250,7 +250,9 @@ describe('DefaultCatalogRulesEnforcer', () => { rules: [ { allow: ['Component'], - locations: [{ type: 'url', match: 'https://github.com/b/**' }], + locations: [ + { type: 'url', pattern: 'https://github.com/b/**' }, + ], }, ], }, diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 38561e0970..9f06a8f0d7 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -30,9 +30,9 @@ export type CatalogRule = { kind: string; }>; locations?: Array<{ - target?: string; + exact?: string; type: string; - match?: string; + pattern?: string; }>; }; @@ -78,6 +78,10 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { * catalog: * rules: * - allow: [Component, API] + * - allow: [Template] + * locations: + * - type: url + * pattern: https://github.com/org/*\/blob/master/template.yaml * * locations: * - type: url @@ -102,13 +106,13 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { .getOptionalConfigArray('locations') ?.map(locationConfig => { const location = { - match: locationConfig.getOptionalString('match'), + pattern: locationConfig.getOptionalString('pattern'), type: locationConfig.getString('type'), - target: locationConfig.getOptionalString('target'), + exact: locationConfig.getOptionalString('exact'), }; - if (location.match && location.target) { + if (location.pattern && location.exact) { throw new Error( - 'A catalog rule location cannot have both target and match values', + 'A catalog rule location cannot have both exact and pattern values', ); } return location; @@ -127,11 +131,11 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return []; } const type = locConf.getString('type'); - const target = resolveTarget(type, locConf.getString('target')); + const exact = resolveTarget(type, locConf.getString('target')); return locConf.getConfigArray('rules').map(ruleConf => ({ allow: ruleConf.getStringArray('allow').map(kind => ({ kind })), - locations: [{ type, target }], + locations: [{ type, exact }], })); }); @@ -163,7 +167,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { private matchLocation( location: LocationSpec, - matchers?: { target?: string; type: string; match?: string }[], + matchers?: { exact?: string; type: string; pattern?: string }[], ): boolean { if (!matchers) { return true; @@ -173,12 +177,12 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (matcher.type !== location?.type) { continue; } - if (matcher.target && matcher.target !== location?.target) { + if (matcher.exact && matcher.exact !== location?.target) { continue; } if ( - matcher.match && - !minimatch(location?.target, matcher.match, { nocase: true }) + matcher.pattern && + !minimatch(location?.target, matcher.pattern, { nocase: true }) ) { continue; } From dbd2480a1a7b6d1489e34b3a755ad751cbc84dd1 Mon Sep 17 00:00:00 2001 From: Esther Annorzie Date: Wed, 30 Nov 2022 12:22:18 -0500 Subject: [PATCH 046/437] Test user provided message if no entities are starred Signed-off-by: Esther Annorzie --- .../StarredEntities/Content.test.tsx | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx b/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx index b761e89e6e..695406aab5 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx @@ -112,4 +112,32 @@ describe('StarredEntitiesContent', () => { getByText('Click the star beside an entity name to add it to this list!'), ).toBeInTheDocument(); }); + + it('should display user provided message if no entities are starred', async () => { + const mockedApi = new MockStarredEntitiesApi(); + + const mockCatalogApi = { + getEntities: jest + .fn() + .mockImplementation(async () => ({ items: entities })), + }; + + const { getByText } = await renderInTestApp( + + + , + { + mountedRoutes: { + '/catalog/:namespace/:kind/:name': entityRouteRef, + }, + }, + ); + + expect(getByText('foo')).toBeInTheDocument(); + }); }); From 446d93f4aff862bba5db961890ebecfb3d176c61 Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Wed, 30 Nov 2022 11:22:28 -0600 Subject: [PATCH 047/437] adding another example Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- plugins/catalog-backend/src/ingestion/CatalogRules.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 9f06a8f0d7..677649d4b2 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -82,6 +82,10 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { * locations: * - type: url * pattern: https://github.com/org/*\/blob/master/template.yaml + * - allow: [Location] + * locations: + * - type: url + * pattern: https://github.com/org/repo/blob/master/location.yaml * * locations: * - type: url From b56bfd12debed9800634df6e6d6e0092e1c4578b Mon Sep 17 00:00:00 2001 From: Juan Pablo Garcia Ripa Date: Mon, 28 Nov 2022 18:10:10 +0100 Subject: [PATCH 048/437] add cofig options to allow warnings and omit messages Signed-off-by: Juan Pablo Garcia Ripa --- packages/repo-tools/cli-report.md | 3 ++ packages/repo-tools/package.json | 1 + .../src/commands/api-reports/api-extractor.ts | 51 +++++++++++-------- .../src/commands/api-reports/api-reports.ts | 22 ++++++-- packages/repo-tools/src/commands/index.ts | 13 +++++ packages/repo-tools/src/lib/paths.ts | 20 ++++++++ yarn.lock | 1 + 7 files changed, 86 insertions(+), 25 deletions(-) create mode 100644 packages/repo-tools/src/lib/paths.ts diff --git a/packages/repo-tools/cli-report.md b/packages/repo-tools/cli-report.md index 309ddad2a3..b2a171cf42 100644 --- a/packages/repo-tools/cli-report.md +++ b/packages/repo-tools/cli-report.md @@ -26,6 +26,9 @@ Options: --ci --tsc --docs + --allow-warnings [allowWarningsPaths...] + --folders + --omitMessages -h, --help ``` diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index a0caa3ee9c..b779b0798f 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -30,6 +30,7 @@ "backstage-repo-tools": "bin/backstage-repo-tools" }, "dependencies": { + "@backstage/cli-common": "workspace:^", "@backstage/errors": "workspace:^", "@manypkg/get-packages": "^1.1.3", "@microsoft/api-documenter": "^7.17.11", diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index bae62ccc7e..9f4f9b238d 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -65,9 +65,10 @@ import { } from '@microsoft/api-documenter/lib/markdown/CustomMarkdownEmitter'; import { IMarkdownEmitterContext } from '@microsoft/api-documenter/lib/markdown/MarkdownEmitter'; import { AstDeclaration } from '@microsoft/api-extractor/lib/analyzer/AstDeclaration'; +import { paths as cliPaths } from '../../lib/paths'; const tmpDir = resolvePath( - process.cwd(), + cliPaths.targetRoot, './node_modules/.cache/api-extractor', ); @@ -219,21 +220,10 @@ ApiReportGenerator.generateReviewFileContent = }); }; -const PACKAGE_ROOTS = ['packages', 'plugins']; - -const ALLOW_WARNINGS = [ - 'packages/core-components', - 'plugins/catalog', - 'plugins/catalog-import', - 'plugins/git-release-manager', - 'plugins/jenkins', - 'plugins/kubernetes', -]; - async function resolvePackagePath( packagePath: string, ): Promise { - const projectRoot = resolvePath(process.cwd()); + const projectRoot = resolvePath(cliPaths.targetRoot); const fullPackageDir = resolvePath(projectRoot, packagePath); const stat = await fs.stat(fullPackageDir); @@ -269,11 +259,11 @@ export async function findSpecificPackageDirs(unresolvedPackageDirs: string[]) { return packageDirs; } -export async function findPackageDirs() { +export async function findPackageDirs(packageRoots: string[]) { const packageDirs = new Array(); - const projectRoot = resolvePath(process.cwd()); + const projectRoot = resolvePath(cliPaths.targetRoot); - for (const packageRoot of PACKAGE_ROOTS) { + for (const packageRoot of packageRoots) { const dirs = await fs.readdir(resolvePath(projectRoot, packageRoot)); for (const dir of dirs) { const packageDir = await resolvePackagePath(join(packageRoot, dir)); @@ -289,7 +279,7 @@ export async function findPackageDirs() { } export async function createTemporaryTsConfig(includedPackageDirs: string[]) { - const path = resolvePath(process.cwd(), 'tsconfig.tmp.json'); + const path = resolvePath(cliPaths.targetRoot, 'tsconfig.tmp.json'); process.once('exit', () => { fs.removeSync(path); @@ -375,6 +365,8 @@ interface ApiExtractionOptions { outputDir: string; isLocalBuild: boolean; tsconfigFilePath: string; + allowWarnings: boolean | string[]; + omitMessages?: string[]; } export async function runApiExtraction({ @@ -382,25 +374,39 @@ export async function runApiExtraction({ outputDir, isLocalBuild, tsconfigFilePath, + allowWarnings, + omitMessages = [], }: ApiExtractionOptions) { await fs.remove(outputDir); const entryPoints = packageDirs.map(packageDir => { return resolvePath( - process.cwd(), + cliPaths.targetRoot, `./dist-types/${packageDir}/src/index.d.ts`, ); }); let compilerState: CompilerState | undefined = undefined; + const allowWarningPkg = Array.isArray(allowWarnings) ? allowWarnings : []; + + const messagesConf: { [key: string]: { logLevel: string } } = {}; + for (const messageCode of omitMessages) { + messagesConf[messageCode] = { + logLevel: 'none', + }; + } const warnings = new Array(); for (const packageDir of packageDirs) { console.log(`## Processing ${packageDir}`); - const projectFolder = resolvePath(process.cwd(), packageDir); + const noBail = Array.isArray(allowWarnings) + ? allowWarnings.includes(packageDir) + : allowWarnings; + + const projectFolder = resolvePath(cliPaths.targetRoot, packageDir); const packageFolder = resolvePath( - process.cwd(), + cliPaths.targetRoot, './dist-types', packageDir, ); @@ -453,6 +459,7 @@ export async function runApiExtraction({ logLevel: 'warning' as ExtractorLogLevel.Warning, addToApiReportFile: true, }, + ...messagesConf, }, tsdocMessageReporting: { default: { @@ -543,12 +550,12 @@ export async function runApiExtraction({ } const warningCountAfter = await countApiReportWarnings(projectFolder); - if (warningCountAfter > 0 && !ALLOW_WARNINGS.includes(packageDir)) { + if (warningCountAfter > 0 && !noBail) { throw new Error( `The API Report for ${packageDir} is not allowed to have warnings`, ); } - if (warningCountAfter === 0 && ALLOW_WARNINGS.includes(packageDir)) { + if (warningCountAfter === 0 && allowWarningPkg.includes(packageDir)) { console.log( `No need to allow warnings for ${packageDir}, it does not have any`, ); diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index 3ac740d596..c66d084d64 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -27,16 +27,29 @@ import { runCliExtraction, buildDocs, } from './api-extractor'; +import { paths as cliPaths } from '../../lib/paths'; export default async (paths: string[], opts: OptionValues) => { + console.log(opts); + console.log({ + ownDir: cliPaths.ownDir, + ownRoot: cliPaths.ownRoot, + targetDir: cliPaths.targetDir, + targetRoot: cliPaths.targetRoot, + 'process.cwd()': process.cwd(), + }); const tmpDir = resolvePath( - process.cwd(), + cliPaths.targetRoot, './node_modules/.cache/api-extractor', ); - const projectRoot = resolvePath(process.cwd()); + + const projectRoot = resolvePath(cliPaths.targetRoot); const isCiBuild = opts.ci; const isDocsBuild = opts.docs; const runTsc = opts.tsc; + const packageRoots = opts.folders; + const allowWarnings: boolean | string[] = opts.allowWarnings; + const omitMessages = opts.omitMessages; const selectedPackageDirs = await findSpecificPackageDirs(paths); @@ -85,7 +98,8 @@ export default async (paths: string[], opts: OptionValues) => { } } - const packageDirs = selectedPackageDirs ?? (await findPackageDirs()); + const packageDirs = + selectedPackageDirs ?? (await findPackageDirs(packageRoots)); const { tsPackageDirs, cliPackageDirs } = await categorizePackageDirs( projectRoot, @@ -99,6 +113,8 @@ export default async (paths: string[], opts: OptionValues) => { outputDir: tmpDir, isLocalBuild: !isCiBuild, tsconfigFilePath, + allowWarnings, + omitMessages, }); } if (cliPackageDirs.length > 0) { diff --git a/packages/repo-tools/src/commands/index.ts b/packages/repo-tools/src/commands/index.ts index ebbcb5be8e..5821dfdd3d 100644 --- a/packages/repo-tools/src/commands/index.ts +++ b/packages/repo-tools/src/commands/index.ts @@ -24,6 +24,19 @@ export function registerCommands(program: Command) { .option('--ci', 'CI run checks that there is no changes on API reports') .option('--tsc', 'executes the tsc compilation before extracting the APIs') .option('--docs', 'generates the api documentation') + .option( + '--allow-warnings [allowWarningsPaths...]', + 'continue processing packages after getting errors on selected packages', + false, + ) + .option('--folders ', 'packages folder containers', [ + 'packages', + 'plugins', + ]) + .option( + '--omitMessages ', + 'select some message code to be omited on the API Extractor (i.e ae-cyclic-inherit-doc)', + ) .description('Generate an API report for selected packages') .action( lazy(() => import('./api-reports/api-reports').then(m => m.default)), diff --git a/packages/repo-tools/src/lib/paths.ts b/packages/repo-tools/src/lib/paths.ts new file mode 100644 index 0000000000..2c658c27b3 --- /dev/null +++ b/packages/repo-tools/src/lib/paths.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { findPaths } from '@backstage/cli-common'; + +/* eslint-disable-next-line no-restricted-syntax */ +export const paths = findPaths(__dirname); diff --git a/yarn.lock b/yarn.lock index 441430dd19..e39514bb98 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8467,6 +8467,7 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/repo-tools@workspace:packages/repo-tools" dependencies: + "@backstage/cli-common": "workspace:^" "@backstage/errors": "workspace:^" "@manypkg/get-packages": ^1.1.3 "@microsoft/api-documenter": ^7.17.11 From 1176a73050b45422d42cef1a0dc66eb88b31c104 Mon Sep 17 00:00:00 2001 From: Juan Pablo Garcia Ripa Date: Wed, 30 Nov 2022 23:51:05 +0100 Subject: [PATCH 049/437] use package.json workspace package as default roots Signed-off-by: Juan Pablo Garcia Ripa --- package.json | 2 +- packages/repo-tools/cli-report.md | 5 +- packages/repo-tools/package.json | 4 + .../src/commands/api-reports/api-extractor.ts | 73 ++++++++----------- .../src/commands/api-reports/api-reports.ts | 38 +++++----- packages/repo-tools/src/commands/index.ts | 10 +-- yarn.lock | 9 +++ 7 files changed, 69 insertions(+), 72 deletions(-) diff --git a/package.json b/package.json index 524cdf4bbe..497a652cdf 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "build:backend": "yarn workspace backend build", "build:all": "backstage-cli repo build --all", "build:api-reports": "yarn build:api-reports:only --tsc", - "build:api-reports:only": "backstage-repo-tools api-reports", + "build:api-reports:only": "backstage-repo-tools api-reports --allow-warnings packages/core-components plugins/catalog plugins/catalog-import plugins/git-release-manager plugins/jenkins plugins/kubernetes", "build:api-docs": "LANG=en_EN yarn build:api-reports --docs", "tsc": "tsc", "tsc:full": "backstage-cli repo clean && tsc --skipLibCheck false --incremental false", diff --git a/packages/repo-tools/cli-report.md b/packages/repo-tools/cli-report.md index b2a171cf42..ba15ecc674 100644 --- a/packages/repo-tools/cli-report.md +++ b/packages/repo-tools/cli-report.md @@ -12,7 +12,7 @@ Options: -h, --help Commands: - api-reports [options] [path...] + api-reports [options] [paths...] type-deps help [command] ``` @@ -20,14 +20,13 @@ Commands: ### `backstage-repo-tools api-reports` ``` -Usage: backstage-repo-tools api-reports [options] [path...] +Usage: backstage-repo-tools api-reports [options] [paths...] Options: --ci --tsc --docs --allow-warnings [allowWarningsPaths...] - --folders --omitMessages -h, --help ``` diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index b779b0798f..657bbb4404 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -40,8 +40,12 @@ "chalk": "^4.0.0", "commander": "^9.1.0", "fs-extra": "10.1.0", + "is-glob": "^4.0.3", "ts-node": "^10.0.0" }, + "devDependencies": { + "@types/is-glob": "^4.0.2" + }, "files": [ "bin", "dist/**/*.js" diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index 9f4f9b238d..d1e8d86240 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -67,8 +67,14 @@ import { IMarkdownEmitterContext } from '@microsoft/api-documenter/lib/markdown/ import { AstDeclaration } from '@microsoft/api-extractor/lib/analyzer/AstDeclaration'; import { paths as cliPaths } from '../../lib/paths'; -const tmpDir = resolvePath( - cliPaths.targetRoot, +import g from 'glob'; +import isGlob from 'is-glob'; + +import { promisify } from 'util'; + +const glob = promisify(g); + +const tmpDir = cliPaths.resolveTargetRoot( './node_modules/.cache/api-extractor', ); @@ -220,11 +226,10 @@ ApiReportGenerator.generateReviewFileContent = }); }; -async function resolvePackagePath( +export async function resolvePackagePath( packagePath: string, ): Promise { - const projectRoot = resolvePath(cliPaths.targetRoot); - const fullPackageDir = resolvePath(projectRoot, packagePath); + const fullPackageDir = cliPaths.resolveTargetRoot(packagePath); const stat = await fs.stat(fullPackageDir); if (!stat.isDirectory()) { @@ -237,49 +242,29 @@ async function resolvePackagePath( } catch (_) { return undefined; } - - return relativePath(projectRoot, fullPackageDir); + return relativePath(cliPaths.targetRoot, fullPackageDir); } -export async function findSpecificPackageDirs(unresolvedPackageDirs: string[]) { +export async function findPackageDirs(selectedPaths: string[]) { const packageDirs = new Array(); + for (const packageRoot of selectedPaths) { + const fullPath = cliPaths.resolveTargetRoot(packageRoot); - for (const unresolvedPackageDir of unresolvedPackageDirs) { - const packageDir = await resolvePackagePath(unresolvedPackageDir); - if (!packageDir) { - throw new Error(`'${unresolvedPackageDir}' is not a valid package path`); - } - packageDirs.push(packageDir); - } - - if (packageDirs.length === 0) { - return undefined; - } - - return packageDirs; -} - -export async function findPackageDirs(packageRoots: string[]) { - const packageDirs = new Array(); - const projectRoot = resolvePath(cliPaths.targetRoot); - - for (const packageRoot of packageRoots) { - const dirs = await fs.readdir(resolvePath(projectRoot, packageRoot)); + // if the path contain any glob notation we resolve all the paths to process one by one + const dirs = isGlob(fullPath) ? await glob(fullPath) : [fullPath]; for (const dir of dirs) { - const packageDir = await resolvePackagePath(join(packageRoot, dir)); + const packageDir = await resolvePackagePath(dir); if (!packageDir) { continue; } - packageDirs.push(packageDir); } } - return packageDirs; } export async function createTemporaryTsConfig(includedPackageDirs: string[]) { - const path = resolvePath(cliPaths.targetRoot, 'tsconfig.tmp.json'); + const path = cliPaths.resolveTargetRoot('tsconfig.tmp.json'); process.once('exit', () => { fs.removeSync(path); @@ -380,8 +365,7 @@ export async function runApiExtraction({ await fs.remove(outputDir); const entryPoints = packageDirs.map(packageDir => { - return resolvePath( - cliPaths.targetRoot, + return cliPaths.resolveTargetRoot( `./dist-types/${packageDir}/src/index.d.ts`, ); }); @@ -404,9 +388,8 @@ export async function runApiExtraction({ ? allowWarnings.includes(packageDir) : allowWarnings; - const projectFolder = resolvePath(cliPaths.targetRoot, packageDir); - const packageFolder = resolvePath( - cliPaths.targetRoot, + const projectFolder = cliPaths.resolveTargetRoot(packageDir); + const packageFolder = cliPaths.resolveTargetRoot( './dist-types', packageDir, ); @@ -1138,10 +1121,7 @@ export async function buildDocs({ documenter.generateFiles(); } -export async function categorizePackageDirs( - projectRoot: string, - packageDirs: any[], -) { +export async function categorizePackageDirs(packageDirs: any[]) { const dirs = packageDirs.slice(); const tsPackageDirs = new Array(); const cliPackageDirs = new Array(); @@ -1157,7 +1137,7 @@ export async function categorizePackageDirs( } const pkgJson = await fs - .readJson(resolvePath(projectRoot, dir, 'package.json')) + .readJson(cliPaths.resolveTargetRoot(dir, 'package.json')) .catch(error => { if (error.code === 'ENOENT') { return undefined; @@ -1211,6 +1191,7 @@ function parseHelpPage(helpPageContent: string) { let options = new Array(); let commands = new Array(); + let commandArguments = new Array(); while (lines.length > 0) { while (lines.length > 0 && !lines[0].endsWith(':')) { @@ -1235,6 +1216,8 @@ function parseHelpPage(helpPageContent: string) { options = sectionItems; } else if (sectionName?.toLocaleLowerCase('en-US') === 'commands:') { commands = sectionItems; + } else if (sectionName?.toLocaleLowerCase('en-US') === 'arguments:') { + commandArguments = sectionItems; } else { throw new Error(`Unknown CLI section: ${sectionName}`); } @@ -1245,6 +1228,7 @@ function parseHelpPage(helpPageContent: string) { usage, options, commands, + commandArguments, }; } @@ -1256,6 +1240,7 @@ interface CliHelpPage { usage: string | undefined; options: string[]; commands: string[]; + commandArguments: string[]; } async function exploreCliHelpPages( @@ -1335,7 +1320,7 @@ export async function runCliExtraction({ }: CliExtractionOptions) { for (const packageDir of packageDirs) { console.log(`## Processing ${packageDir}`); - const fullDir = resolvePath(projectRoot, packageDir); + const fullDir = cliPaths.resolveTargetRoot(packageDir); const pkgJson = await fs.readJson(resolvePath(fullDir, 'package.json')); if (!pkgJson.bin) { diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index c66d084d64..9b0d78d6a8 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -19,7 +19,6 @@ import { resolve as resolvePath } from 'path'; import fs from 'fs-extra'; import { spawnSync } from 'child_process'; import { - findSpecificPackageDirs, createTemporaryTsConfig, findPackageDirs, categorizePackageDirs, @@ -30,14 +29,6 @@ import { import { paths as cliPaths } from '../../lib/paths'; export default async (paths: string[], opts: OptionValues) => { - console.log(opts); - console.log({ - ownDir: cliPaths.ownDir, - ownRoot: cliPaths.ownRoot, - targetDir: cliPaths.targetDir, - targetRoot: cliPaths.targetRoot, - 'process.cwd()': process.cwd(), - }); const tmpDir = resolvePath( cliPaths.targetRoot, './node_modules/.cache/api-extractor', @@ -47,25 +38,26 @@ export default async (paths: string[], opts: OptionValues) => { const isCiBuild = opts.ci; const isDocsBuild = opts.docs; const runTsc = opts.tsc; - const packageRoots = opts.folders; + const selectedPaths = paths.length ? paths : await getWorkspacePkgs(); const allowWarnings: boolean | string[] = opts.allowWarnings; const omitMessages = opts.omitMessages; - const selectedPackageDirs = await findSpecificPackageDirs(paths); + const selectedPackageDirs = await findPackageDirs(selectedPaths); - if (selectedPackageDirs && isCiBuild) { + if (paths.length && isCiBuild) { + // TODO @sarabadu we can remove this validation to allow `/plugins/*` on CI?? throw new Error( 'Package path arguments are not supported together with the --ci flag', ); } - if (!selectedPackageDirs && !isCiBuild && !isDocsBuild) { + if (!paths.length && !isCiBuild && !isDocsBuild) { console.log(''); console.log( 'TIP: You can generate api-reports for select packages by passing package paths:', ); console.log(''); console.log( - ' yarn build:api-reports packages/config packages/core-plugin-api', + ' yarn build:api-reports packages/config packages/core-plugin-api plugins/*', ); console.log(''); } @@ -98,12 +90,8 @@ export default async (paths: string[], opts: OptionValues) => { } } - const packageDirs = - selectedPackageDirs ?? (await findPackageDirs(packageRoots)); - const { tsPackageDirs, cliPackageDirs } = await categorizePackageDirs( - projectRoot, - packageDirs, + selectedPackageDirs, ); if (tsPackageDirs.length > 0) { @@ -134,3 +122,15 @@ export default async (paths: string[], opts: OptionValues) => { }); } }; +async function getWorkspacePkgs() { + const pkgJson = await fs + .readJson(cliPaths.resolveTargetRoot('package.json')) + .catch(error => { + if (error.code === 'ENOENT') { + return undefined; + } + throw error; + }); + const workspaces = pkgJson?.workspaces?.packages; + return workspaces; +} diff --git a/packages/repo-tools/src/commands/index.ts b/packages/repo-tools/src/commands/index.ts index 5821dfdd3d..bbc6773f83 100644 --- a/packages/repo-tools/src/commands/index.ts +++ b/packages/repo-tools/src/commands/index.ts @@ -20,7 +20,11 @@ import { exitWithError } from '../lib/errors'; export function registerCommands(program: Command) { program - .command('api-reports [path...]') + .command('api-reports') + .argument( + '[paths...]', + 'path of package folder to extract API reports, `workspaces.packages` from root packages.json by default', + ) .option('--ci', 'CI run checks that there is no changes on API reports') .option('--tsc', 'executes the tsc compilation before extracting the APIs') .option('--docs', 'generates the api documentation') @@ -29,10 +33,6 @@ export function registerCommands(program: Command) { 'continue processing packages after getting errors on selected packages', false, ) - .option('--folders ', 'packages folder containers', [ - 'packages', - 'plugins', - ]) .option( '--omitMessages ', 'select some message code to be omited on the API Extractor (i.e ae-cyclic-inherit-doc)', diff --git a/yarn.lock b/yarn.lock index e39514bb98..c9a9d1daea 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8474,9 +8474,11 @@ __metadata: "@microsoft/api-extractor": ^7.23.0 "@microsoft/api-extractor-model": ^7.17.2 "@microsoft/tsdoc": 0.14.1 + "@types/is-glob": ^4.0.2 chalk: ^4.0.0 commander: ^9.1.0 fs-extra: 10.1.0 + is-glob: ^4.0.3 ts-node: ^10.0.0 bin: backstage-repo-tools: bin/backstage-repo-tools @@ -14201,6 +14203,13 @@ __metadata: languageName: node linkType: hard +"@types/is-glob@npm:^4.0.2": + version: 4.0.2 + resolution: "@types/is-glob@npm:4.0.2" + checksum: 50b0a52b6d179781b36bfce35155e1e0dc66b62e2943153d7d7c7079c40ba6236528a254de8be6c52ff9a7a351996887802efd7fd763da3e2121315e4ffe2edf + languageName: node + linkType: hard + "@types/istanbul-lib-coverage@npm:*, @types/istanbul-lib-coverage@npm:^2.0.0, @types/istanbul-lib-coverage@npm:^2.0.1": version: 2.0.1 resolution: "@types/istanbul-lib-coverage@npm:2.0.1" From a8611bcac44753c1e0144b7333226c9b3c2e7faf Mon Sep 17 00:00:00 2001 From: Juan Pablo Garcia Ripa Date: Thu, 1 Dec 2022 14:16:17 +0100 Subject: [PATCH 050/437] add changeset Signed-off-by: Juan Pablo Garcia Ripa --- .changeset/lemon-coats-camp.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .changeset/lemon-coats-camp.md diff --git a/.changeset/lemon-coats-camp.md b/.changeset/lemon-coats-camp.md new file mode 100644 index 0000000000..a2d87476f6 --- /dev/null +++ b/.changeset/lemon-coats-camp.md @@ -0,0 +1,11 @@ +--- +'@backstage/repo-tools': minor +--- + +Add new command options to the `api-report` + +- added `--allowWarnings` to continue processing packages if some packages have warnings +- added `--omitMessages` to pass some warnings messages code to be omitted from the api-report.md files +- The `paths` argument for this command now takes as default the value on `workspaces.packages` inside the root package.json +- The `paths` argument now allow glob patterns +- change the path resolution to use the `@backstage/cli-common` packages instead From e48fc1f1ae82672151f8ebf1f85de34072693812 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 29 Nov 2022 14:22:47 +0100 Subject: [PATCH 051/437] Add optional, backward compatible logger to PG engine/indexer Signed-off-by: Eric Peterson --- .changeset/search-antibacterial-wipe.md | 12 ++++++++++++ plugins/search-backend-module-pg/api-report.md | 6 +++++- plugins/search-backend-module-pg/package.json | 3 ++- .../src/PgSearchEngine/PgSearchEngine.ts | 14 +++++++++++++- .../src/PgSearchEngine/PgSearchEngineIndexer.ts | 5 +++++ yarn.lock | 1 + 6 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 .changeset/search-antibacterial-wipe.md diff --git a/.changeset/search-antibacterial-wipe.md b/.changeset/search-antibacterial-wipe.md new file mode 100644 index 0000000000..d69edabd3f --- /dev/null +++ b/.changeset/search-antibacterial-wipe.md @@ -0,0 +1,12 @@ +--- +'@backstage/plugin-search-backend-module-pg': minor +--- + +Added the option to pass a logger to `PgSearchEngine` during instantiation. You may do so as follows: + +```diff +const searchEngine = await PgSearchEngine.fromConfig(env.config, { + database: env.database, ++ logger: env.logger, +}); +``` diff --git a/plugins/search-backend-module-pg/api-report.md b/plugins/search-backend-module-pg/api-report.md index d35bac66aa..3c6096090e 100644 --- a/plugins/search-backend-module-pg/api-report.md +++ b/plugins/search-backend-module-pg/api-report.md @@ -8,6 +8,7 @@ import { Config } from '@backstage/config'; import { IndexableDocument } from '@backstage/plugin-search-common'; import { IndexableResultSet } from '@backstage/plugin-search-common'; import { Knex } from 'knex'; +import { Logger } from 'winston'; import { PluginDatabaseManager } from '@backstage/backend-common'; import { SearchEngine } from '@backstage/plugin-search-common'; import { SearchQuery } from '@backstage/plugin-search-common'; @@ -84,11 +85,12 @@ export interface DocumentResultRow { // @public (undocumented) export class PgSearchEngine implements SearchEngine { // @deprecated - constructor(databaseStore: DatabaseStore, config: Config); + constructor(databaseStore: DatabaseStore, config: Config, logger?: Logger); // @deprecated (undocumented) static from(options: { database: PluginDatabaseManager; config: Config; + logger?: Logger; }): Promise; // (undocumented) static fromConfig( @@ -126,6 +128,7 @@ export type PgSearchEngineIndexerOptions = { batchSize: number; type: string; databaseStore: DatabaseStore; + logger?: Logger; }; // @public @@ -144,6 +147,7 @@ export type PgSearchHighlightOptions = { // @public export type PgSearchOptions = { database: PluginDatabaseManager; + logger?: Logger; }; // @public (undocumented) diff --git a/plugins/search-backend-module-pg/package.json b/plugins/search-backend-module-pg/package.json index 25ee3cf696..2e0f46f24a 100644 --- a/plugins/search-backend-module-pg/package.json +++ b/plugins/search-backend-module-pg/package.json @@ -29,7 +29,8 @@ "@backstage/plugin-search-common": "workspace:^", "knex": "^2.0.0", "lodash": "^4.17.21", - "uuid": "^8.3.2" + "uuid": "^8.3.2", + "winston": "^3.2.1" }, "devDependencies": { "@backstage/backend-test-utils": "workspace:^", diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index f53c433c2f..d2cf80e0ca 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -28,6 +28,7 @@ import { PgSearchQuery, } from '../database'; import { v4 as uuid } from 'uuid'; +import { Logger } from 'winston'; import { Config } from '@backstage/config'; /** @@ -62,6 +63,7 @@ export type PgSearchQueryTranslator = ( */ export type PgSearchOptions = { database: PluginDatabaseManager; + logger?: Logger; }; /** @@ -82,12 +84,17 @@ export type PgSearchHighlightOptions = { /** @public */ export class PgSearchEngine implements SearchEngine { + private readonly logger?: Logger; private readonly highlightOptions: PgSearchHighlightOptions; /** * @deprecated This will be marked as private in a future release, please us fromConfig instead */ - constructor(private readonly databaseStore: DatabaseStore, config: Config) { + constructor( + private readonly databaseStore: DatabaseStore, + config: Config, + logger?: Logger, + ) { const uuidTag = uuid(); const highlightConfig = config.getOptionalConfig( 'search.pg.highlightOptions', @@ -107,6 +114,7 @@ export class PgSearchEngine implements SearchEngine { highlightConfig?.getOptionalString('fragmentDelimiter') ?? ' ... ', }; this.highlightOptions = highlightOptions; + this.logger = logger; } /** @@ -115,10 +123,12 @@ export class PgSearchEngine implements SearchEngine { static async from(options: { database: PluginDatabaseManager; config: Config; + logger?: Logger; }): Promise { return new PgSearchEngine( await DatabaseDocumentStore.create(options.database), options.config, + options.logger, ); } @@ -126,6 +136,7 @@ export class PgSearchEngine implements SearchEngine { return new PgSearchEngine( await DatabaseDocumentStore.create(options.database), config, + options.logger, ); } @@ -170,6 +181,7 @@ export class PgSearchEngine implements SearchEngine { batchSize: 1000, type, databaseStore: this.databaseStore, + logger: this.logger?.child({ documentType: type }), }); } diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts index e3dbe82751..63dceec4dc 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts @@ -14,9 +14,11 @@ * limitations under the License. */ +import { getVoidLogger } from '@backstage/backend-common'; import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; import { IndexableDocument } from '@backstage/plugin-search-common'; import { Knex } from 'knex'; +import { Logger } from 'winston'; import { DatabaseStore } from '../database'; /** @public */ @@ -24,10 +26,12 @@ export type PgSearchEngineIndexerOptions = { batchSize: number; type: string; databaseStore: DatabaseStore; + logger?: Logger; }; /** @public */ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { + private logger: Logger; private store: DatabaseStore; private type: string; private tx: Knex.Transaction | undefined; @@ -36,6 +40,7 @@ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { super({ batchSize: options.batchSize }); this.store = options.databaseStore; this.type = options.type; + this.logger = options.logger || getVoidLogger(); } async initialize(): Promise { diff --git a/yarn.lock b/yarn.lock index e3d1b5eb88..c3a2a2a844 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7535,6 +7535,7 @@ __metadata: knex: ^2.0.0 lodash: ^4.17.21 uuid: ^8.3.2 + winston: ^3.2.1 languageName: unknown linkType: soft From 18646ccb1ad767d8c0225dbe50ddf5b510020373 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 29 Nov 2022 15:12:23 +0100 Subject: [PATCH 052/437] Update lunr engine to preserve pre-existing indices if indexer receives 0 documents Signed-off-by: Eric Peterson --- .../src/engines/LunrSearchEngine.test.ts | 56 ++++++++++++++++++- .../src/engines/LunrSearchEngine.ts | 29 +++++++++- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts index 4ef380943f..27cd65d63b 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts @@ -1026,7 +1026,7 @@ describe('LunrSearchEngine', () => { // Get the indexer and invoke its close handler. await inspectableSearchEngine.getIndexer('test-index'); - const onClose = indexerMock.on.mock.calls[0][1] as Function; + const onClose = indexerMock.on.mock.calls[1][1] as Function; onClose(); // Ensure mocked methods were called. @@ -1044,6 +1044,60 @@ describe('LunrSearchEngine', () => { 'new-location': doc, }); }); + + it('should not replace index or docs if no docs were indexed', async () => { + // Set up an inspectable search engine to pre-set some data. + const doc = { title: 'A doc', text: 'test', location: 'some-location' }; + const inspectableSearchEngine = new LunrSearchEngineForTests({ + logger: getVoidLogger(), + }); + inspectableSearchEngine.setDocStore({ 'existing-location': doc }); + + // Mock methods called by close handler (resolving no documents) + indexerMock.buildIndex.mockReturnValueOnce('expected-index'); + indexerMock.getDocumentStore.mockReturnValueOnce({}); + + // Get the indexer and invoke its close handler. + await inspectableSearchEngine.getIndexer('test-index'); + const onClose = indexerMock.on.mock.calls[1][1] as Function; + onClose(); + + // Ensure buildIndex method was not called. + expect(indexerMock.buildIndex).not.toHaveBeenCalled(); + + // Ensure pre-existing documents still exist in the store + expect(inspectableSearchEngine.getDocStore()).toStrictEqual({ + 'existing-location': doc, + }); + }); + + it('should not replace index or docs if an error was thrown', async () => { + // Set up an inspectable search engine to pre-set some data. + const doc = { title: 'A doc', text: 'test', location: 'some-location' }; + const inspectableSearchEngine = new LunrSearchEngineForTests({ + logger: getVoidLogger(), + }); + inspectableSearchEngine.setDocStore({ 'existing-location': doc }); + + // Mock methods called by close handler (resolving no documents) + indexerMock.buildIndex.mockReturnValueOnce('expected-index'); + indexerMock.getDocumentStore.mockReturnValueOnce({}); + + // Get the indexer and invoke its close handler after firing an error. + await inspectableSearchEngine.getIndexer('test-index'); + const onError = indexerMock.on.mock.calls[0][1] as Function; + const onClose = indexerMock.on.mock.calls[1][1] as Function; + onError(new Error('Some collator error')); + onClose(); + + // Ensure buildIndex method was not called. + expect(indexerMock.buildIndex).not.toHaveBeenCalled(); + + // Ensure pre-existing documents still exist in the store + expect(inspectableSearchEngine.getDocStore()).toStrictEqual({ + 'existing-location': doc, + }); + }); }); }); diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts index ff2c828402..b69b5ea863 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts @@ -153,12 +153,37 @@ export class LunrSearchEngine implements SearchEngine { async getIndexer(type: string) { const indexer = new LunrSearchEngineIndexer(); + const indexerLogger = this.logger.child({ documentType: type }); + let errorThrown: Error | undefined; + + indexer.on('error', err => { + errorThrown = err; + }); indexer.on('close', () => { // Once the stream is closed, build the index and store the documents in // memory for later retrieval. - this.lunrIndices[type] = indexer.buildIndex(); - this.docStore = { ...this.docStore, ...indexer.getDocumentStore() }; + const newDocuments = indexer.getDocumentStore(); + const docStoreExists = this.lunrIndices[type] !== undefined; + const documentsIndexed = Object.keys(newDocuments).length; + + // Do not set the index if there was an error or if no documents were + // indexed. This ensures search continues to work for an index, even in + // case of transient issues in underlying collators. + if (!errorThrown && documentsIndexed > 0) { + this.lunrIndices[type] = indexer.buildIndex(); + this.docStore = { ...this.docStore, ...newDocuments }; + } else { + indexerLogger.warn( + `Index for ${type} was not ${ + docStoreExists ? 'replaced' : 'created' + }: ${ + errorThrown + ? 'an error was encountered' + : 'indexer received 0 documents' + }`, + ); + } }); return indexer; From b24cf5a99e4ae027169b4e727947ebb367250f1b Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 29 Nov 2022 15:13:31 +0100 Subject: [PATCH 053/437] Update pg engine to preserve pre-existing indices if indexer receives 0 documents Signed-off-by: Eric Peterson --- .../PgSearchEngine/PgSearchEngineIndexer.test.ts | 9 +++++++++ .../src/PgSearchEngine/PgSearchEngineIndexer.ts | 14 ++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts index 0b9cd96c19..f1cb3923e1 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts @@ -81,6 +81,15 @@ describe('PgSearchEngineIndexer', () => { expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); }); + it('should rollback transaction if no documents indexed', async () => { + await TestPipeline.fromIndexer(indexer).withDocuments([]).execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).not.toHaveBeenCalled(); + expect(database.completeInsert).not.toHaveBeenCalled(); + expect(tx.rollback).toHaveBeenCalled(); + }); + it('should close out stream and bubble up error on prepare', async () => { const expectedError = new Error('Prepare error'); const documents = [ diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts index 63dceec4dc..ba625bc116 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts @@ -35,6 +35,7 @@ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { private store: DatabaseStore; private type: string; private tx: Knex.Transaction | undefined; + private numRecords = 0; constructor(options: PgSearchEngineIndexerOptions) { super({ batchSize: options.batchSize }); @@ -56,6 +57,8 @@ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { } async index(documents: IndexableDocument[]): Promise { + this.numRecords += documents.length; + try { await this.store.insertDocuments(this.tx!, this.type, documents); } catch (e) { @@ -67,6 +70,17 @@ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { } async finalize(): Promise { + // If no documents were indexed, rollback the transaction, log a warning, + // and do not continue. This ensures that collators that return empty sets + // of documents do not cause the index to be deleted. + if (this.numRecords === 0) { + this.logger.warn( + `Index for ${this.type} was not replaced: indexer received 0 documents`, + ); + this.tx!.rollback!(); + return; + } + // Attempt to complete and commit the transaction. try { await this.store.completeInsert(this.tx!, this.type); From f248b75bde5247333e208c37bacd2cfa3c6a55cd Mon Sep 17 00:00:00 2001 From: Clare Liguori Date: Thu, 1 Dec 2022 12:06:32 -0800 Subject: [PATCH 054/437] Rename AwsCredentialsProvider.getCredentials -> AwsCredentialsManager.getCredentialProvider Signed-off-by: Clare Liguori --- packages/integration-aws-node/README.md | 41 +++--- packages/integration-aws-node/api-report.md | 26 ++-- ...s => DefaultAwsCredentialsManager.test.ts} | 138 +++++++++--------- ...der.ts => DefaultAwsCredentialsManager.ts} | 110 +++++++------- packages/integration-aws-node/src/index.ts | 8 +- packages/integration-aws-node/src/types.ts | 21 ++- .../src/stages/publish/awsS3.test.ts | 30 ++-- .../techdocs-node/src/stages/publish/awsS3.ts | 21 +-- 8 files changed, 211 insertions(+), 184 deletions(-) rename packages/integration-aws-node/src/{DefaultAwsCredentialsProvider.test.ts => DefaultAwsCredentialsManager.test.ts} (66%) rename packages/integration-aws-node/src/{DefaultAwsCredentialsProvider.ts => DefaultAwsCredentialsManager.ts} (66%) diff --git a/packages/integration-aws-node/README.md b/packages/integration-aws-node/README.md index 6132f4240a..62df26756f 100644 --- a/packages/integration-aws-node/README.md +++ b/packages/integration-aws-node/README.md @@ -11,7 +11,7 @@ Backstage app config. Users can configure IAM user credentials, IAM roles, and profile names for their AWS accounts in their Backstage config. -If the AWS integration configuration is missing, the credentials provider +If the AWS integration configuration is missing, the credentials manager from this package will fall back to the AWS SDK default credentials chain for resources in the main AWS account. The default credentials chain for Node resolves credentials in the @@ -79,32 +79,35 @@ aws: ## Integrate new plugins Backend plugins can provide an AWS ARN or account ID to this library in order to -retrieve a credentials provider for the relevant account that can be fed directly +retrieve a credential provider for the relevant account that can be fed directly to an AWS SDK client. The AWS SDK for Javascript V3 must be used. ```typescript -const awsCredentialsProvider = DefaultAwsCredentialsProvider.fromConfig(config); +const awsCredentialsManager = DefaultAwsCredentialsManager.fromConfig(config); // provide the account ID explicitly -const creds = await awsCredentialsProvider.getCredentials({ accountId }); +const credProvider = await awsCredentialsManager.getCredentialProvider({ + accountId, +}); // OR extract the account ID from the ARN -const creds = await awsCredentialsProvider.getCredentials({ arn }); +const credProvider = await awsCredentialsManager.getCredentialProvider({ arn }); // OR provide neither to get main account's credentials -const creds = await awsCredentialsProvider.getCredentials({}); +const credProvider = await awsCredentialsManager.getCredentialProvider({}); -// Example constructing an AWS Proton client with the returned credentials provider +// Example constructing an AWS Proton client with the returned credential provider const client = new ProtonClient({ region, - credentialDefaultProvider: () => creds.provider, + credentialDefaultProvider: () => credProvider.sdkCredentialProvider, }); ``` -Depending on the nature of your plguin, you may either have the user specify the +Depending on the nature of your plugin, you may either have the user specify the relevant ARN or account ID in a catalog entity annotation or in the static Backstage app configuration for your plugin. -For example, you can create a new catalog entity annotation for your plugin: +For example, you can create a new catalog entity annotation for your plugin containing +either an AWS account ID or ARN: ```yaml apiVersion: backstage.io/v1alpha1 @@ -117,7 +120,7 @@ metadata: my-other-plugin.io/aws-dynamodb-table: 'arn:aws:dynamodb:us-east-2:123456789012:table/example-table' ``` -In your plugin, read the annotation value so that you can retrieve the credentials provider: +In your plugin, read the annotation value so that you can retrieve the credential provider: ```typescript const MY_AWS_ACCOUNT_ID_ANNOTATION = 'my-plugin.io/aws-account-id'; @@ -126,7 +129,7 @@ const getAwsAccountId = (entity: Entity) => entity.metadata.annotations?.[MY_AWS_ACCOUNT_ID_ANNOTATION]); ``` -Alternatively, you can create a new configuration field for your plugin: +Alternatively, you can create a new Backstage app configuration field for your plugin: ```yaml # app-config.yaml @@ -138,18 +141,20 @@ my-other-plugin: awsDynamoDbTable: 'arn:aws:dynamodb:us-east-2:123456789012:table/example-table' ``` -In your plugin, read the configuration value so that you can retrieve the credentials provider: +In your plugin, read the configuration value so that you can retrieve the credential provider: ```typescript // Read an account ID from your plugin's configuration -const awsCredentialsProvider = DefaultAwsCredentialsProvider.fromConfig(config); -const accountId = config.getString('my-plugin.awsAccountId'); -const creds = await awsCredentialsProvider.getCredentials({ accountId }); +const awsCredentialsManager = DefaultAwsCredentialsManager.fromConfig(config); +const accountId = config.getOptionalString('my-plugin.awsAccountId'); +const credProvider = await awsCredentialsManager.getCredentialProvider({ + accountId, +}); // Or, read an AWS ARN from your plugin's configuration -const awsCredentialsProvider = DefaultAwsCredentialsProvider.fromConfig(config); +const awsCredentialsManager = DefaultAwsCredentialsManager.fromConfig(config); const arn = config.getString('my-other-plugin.awsDynamoDbTable'); -const creds = await awsCredentialsProvider.getCredentials({ arn }); +const credProvider = await awsCredentialsManager.getCredentialProvider({ arn }); ``` ## Links diff --git a/packages/integration-aws-node/api-report.md b/packages/integration-aws-node/api-report.md index bac43c67d4..58bdde4090 100644 --- a/packages/integration-aws-node/api-report.md +++ b/packages/integration-aws-node/api-report.md @@ -7,23 +7,25 @@ import { AwsCredentialIdentityProvider } from '@aws-sdk/types'; import { Config } from '@backstage/config'; // @public -export type AwsCredentials = { +export type AwsCredentialProvider = { accountId?: string; stsRegion?: string; - provider: AwsCredentialIdentityProvider; + sdkCredentialProvider: AwsCredentialIdentityProvider; }; // @public -export interface AwsCredentialsProvider { - getCredentials(opts?: AwsCredentialsProviderOptions): Promise; -} - -// @public -export type AwsCredentialsProviderOptions = { +export type AwsCredentialProviderOptions = { accountId?: string; arn?: string; }; +// @public +export interface AwsCredentialsManager { + getCredentialProvider( + opts?: AwsCredentialProviderOptions, + ): Promise; +} + // @public export type AwsIntegrationAccountConfig = { accountId: string; @@ -60,10 +62,12 @@ export type AwsIntegrationMainAccountConfig = { }; // @public -export class DefaultAwsCredentialsProvider implements AwsCredentialsProvider { +export class DefaultAwsCredentialsManager implements AwsCredentialsManager { // (undocumented) - static fromConfig(config: Config): DefaultAwsCredentialsProvider; - getCredentials(opts?: AwsCredentialsProviderOptions): Promise; + static fromConfig(config: Config): DefaultAwsCredentialsManager; + getCredentialProvider( + opts?: AwsCredentialProviderOptions, + ): Promise; } // @public diff --git a/packages/integration-aws-node/src/DefaultAwsCredentialsProvider.test.ts b/packages/integration-aws-node/src/DefaultAwsCredentialsManager.test.ts similarity index 66% rename from packages/integration-aws-node/src/DefaultAwsCredentialsProvider.test.ts rename to packages/integration-aws-node/src/DefaultAwsCredentialsManager.test.ts index 60f2f558b8..1b21d43338 100644 --- a/packages/integration-aws-node/src/DefaultAwsCredentialsProvider.test.ts +++ b/packages/integration-aws-node/src/DefaultAwsCredentialsManager.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { DefaultAwsCredentialsProvider } from './DefaultAwsCredentialsProvider'; +import { DefaultAwsCredentialsManager } from './DefaultAwsCredentialsManager'; import { mockClient, AwsClientStub } from 'aws-sdk-client-mock'; import 'aws-sdk-client-mock-jest'; import { @@ -31,7 +31,7 @@ let config: Config; jest.mock('fs', () => ({ promises: { readFile: jest.fn() } })); -describe('DefaultAwsCredentialsProvider', () => { +describe('DefaultAwsCredentialsManager', () => { beforeEach(() => { process.env = { ...env }; jest.resetAllMocks(); @@ -145,16 +145,16 @@ describe('DefaultAwsCredentialsProvider', () => { process.env = env; }); - describe('#getCredentials', () => { + describe('#getCredentialProvider', () => { it('retrieves assume-role creds for the given account ID and caches the provider', async () => { - const provider = DefaultAwsCredentialsProvider.fromConfig(config); - const awsCredentials = await provider.getCredentials({ + const provider = DefaultAwsCredentialsManager.fromConfig(config); + const awsCredentialProvider = await provider.getCredentialProvider({ accountId: '111111111111', }); - expect(awsCredentials.accountId).toEqual('111111111111'); + expect(awsCredentialProvider.accountId).toEqual('111111111111'); - const creds = await awsCredentials.provider(); + const creds = await awsCredentialProvider.sdkCredentialProvider(); expect(creds).toEqual({ accessKeyId: 'ACCESS_KEY_ID_1', secretAccessKey: 'SECRET_ACCESS_KEY_1', @@ -162,23 +162,23 @@ describe('DefaultAwsCredentialsProvider', () => { expiration: new Date('2022-01-01'), }); - const awsCredentials2 = await provider.getCredentials({ + const awsCredentialProvider2 = await provider.getCredentialProvider({ accountId: '111111111111', }); - expect(awsCredentials).toBe(awsCredentials2); + expect(awsCredentialProvider).toBe(awsCredentialProvider2); expect(stsMock).toHaveReceivedCommandTimes(AssumeRoleCommand, 1); }); it('retrieves assume-role creds in another partition for the given account ID', async () => { - const provider = DefaultAwsCredentialsProvider.fromConfig(config); - const awsCredentials = await provider.getCredentials({ + const provider = DefaultAwsCredentialsManager.fromConfig(config); + const awsCredentialProvider = await provider.getCredentialProvider({ accountId: '222222222222', }); - expect(awsCredentials.accountId).toEqual('222222222222'); + expect(awsCredentialProvider.accountId).toEqual('222222222222'); - const creds = await awsCredentials.provider(); + const creds = await awsCredentialProvider.sdkCredentialProvider(); expect(creds).toEqual({ accessKeyId: 'ACCESS_KEY_ID_2', secretAccessKey: 'SECRET_ACCESS_KEY_2', @@ -188,14 +188,14 @@ describe('DefaultAwsCredentialsProvider', () => { }); it('retrieves assume-role creds for an account using the account defaults', async () => { - const provider = DefaultAwsCredentialsProvider.fromConfig(config); - const awsCredentials = await provider.getCredentials({ + const provider = DefaultAwsCredentialsManager.fromConfig(config); + const awsCredentialProvider = await provider.getCredentialProvider({ accountId: '999999999999', }); - expect(awsCredentials.accountId).toEqual('999999999999'); + expect(awsCredentialProvider.accountId).toEqual('999999999999'); - const creds = await awsCredentials.provider(); + const creds = await awsCredentialProvider.sdkCredentialProvider(); expect(creds).toEqual({ accessKeyId: 'ACCESS_KEY_ID_9', secretAccessKey: 'SECRET_ACCESS_KEY_9', @@ -205,14 +205,14 @@ describe('DefaultAwsCredentialsProvider', () => { }); it('retrieves static creds for the given account ID', async () => { - const provider = DefaultAwsCredentialsProvider.fromConfig(config); - const awsCredentials = await provider.getCredentials({ + const provider = DefaultAwsCredentialsManager.fromConfig(config); + const awsCredentialProvider = await provider.getCredentialProvider({ accountId: '333333333333', }); - expect(awsCredentials.accountId).toEqual('333333333333'); + expect(awsCredentialProvider.accountId).toEqual('333333333333'); - const creds = await awsCredentials.provider(); + const creds = await awsCredentialProvider.sdkCredentialProvider(); expect(creds).toEqual({ accessKeyId: 'my-access-key', secretAccessKey: 'my-secret-access-key', @@ -228,14 +228,14 @@ describe('DefaultAwsCredentialsProvider', () => { }, }, }); - const provider = DefaultAwsCredentialsProvider.fromConfig(minConfig); - const awsCredentials = await provider.getCredentials({ + const provider = DefaultAwsCredentialsManager.fromConfig(minConfig); + const awsCredentialProvider = await provider.getCredentialProvider({ accountId: '123456789012', }); - expect(awsCredentials.accountId).toEqual('123456789012'); + expect(awsCredentialProvider.accountId).toEqual('123456789012'); - const creds = await awsCredentials.provider(); + const creds = await awsCredentialProvider.sdkCredentialProvider(); expect(creds).toEqual({ accessKeyId: 'GHI', secretAccessKey: 'JKL', @@ -251,23 +251,23 @@ describe('DefaultAwsCredentialsProvider', () => { }, }, }); - const provider = DefaultAwsCredentialsProvider.fromConfig(minConfig); - const awsCredentials1 = await provider.getCredentials({}); - const awsCredentials2 = await provider.getCredentials({}); + const provider = DefaultAwsCredentialsManager.fromConfig(minConfig); + const awsCredentialProvider1 = await provider.getCredentialProvider({}); + const awsCredentialProvider2 = await provider.getCredentialProvider({}); - expect(awsCredentials1).toBe(awsCredentials2); + expect(awsCredentialProvider1).toBe(awsCredentialProvider2); expect(stsMock).toHaveReceivedCommandTimes(GetCallerIdentityCommand, 1); }); it('retrieves the ini provider chain for the given account ID', async () => { - const provider = DefaultAwsCredentialsProvider.fromConfig(config); - const awsCredentials = await provider.getCredentials({ + const provider = DefaultAwsCredentialsManager.fromConfig(config); + const awsCredentialProvider = await provider.getCredentialProvider({ accountId: '555555555555', }); - expect(awsCredentials.accountId).toEqual('555555555555'); + expect(awsCredentialProvider.accountId).toEqual('555555555555'); - const creds = await awsCredentials.provider(); + const creds = await awsCredentialProvider.sdkCredentialProvider(); expect(creds).toEqual({ accessKeyId: 'ACCESS_KEY_ID_9', secretAccessKey: 'SECRET_ACCESS_KEY_9', @@ -275,14 +275,14 @@ describe('DefaultAwsCredentialsProvider', () => { }); it('retrieves the default cred provider chain for the given account ID', async () => { - const provider = DefaultAwsCredentialsProvider.fromConfig(config); - const awsCredentials = await provider.getCredentials({ + const provider = DefaultAwsCredentialsManager.fromConfig(config); + const awsCredentialProvider = await provider.getCredentialProvider({ accountId: '444444444444', }); - expect(awsCredentials.accountId).toEqual('444444444444'); + expect(awsCredentialProvider.accountId).toEqual('444444444444'); - const creds = await awsCredentials.provider(); + const creds = await awsCredentialProvider.sdkCredentialProvider(); expect(creds).toEqual({ accessKeyId: 'ACCESS_KEY_ID_10', secretAccessKey: 'SECRET_ACCESS_KEY_10', @@ -299,14 +299,14 @@ describe('DefaultAwsCredentialsProvider', () => { }, }, }); - const provider = DefaultAwsCredentialsProvider.fromConfig(minConfig); - const awsCredentials = await provider.getCredentials({ + const provider = DefaultAwsCredentialsManager.fromConfig(minConfig); + const awsCredentialProvider = await provider.getCredentialProvider({ accountId: '123456789012', }); - expect(awsCredentials.accountId).toEqual('123456789012'); + expect(awsCredentialProvider.accountId).toEqual('123456789012'); - const creds = await awsCredentials.provider(); + const creds = await awsCredentialProvider.sdkCredentialProvider(); expect(creds).toEqual({ accessKeyId: 'ACCESS_KEY_ID_9', secretAccessKey: 'SECRET_ACCESS_KEY_9', @@ -317,14 +317,14 @@ describe('DefaultAwsCredentialsProvider', () => { const minConfig = new ConfigReader({ aws: {}, }); - const provider = DefaultAwsCredentialsProvider.fromConfig(minConfig); - const awsCredentials = await provider.getCredentials({ + const provider = DefaultAwsCredentialsManager.fromConfig(minConfig); + const awsCredentialProvider = await provider.getCredentialProvider({ accountId: '123456789012', }); - expect(awsCredentials.accountId).toEqual('123456789012'); + expect(awsCredentialProvider.accountId).toEqual('123456789012'); - const creds = await awsCredentials.provider(); + const creds = await awsCredentialProvider.sdkCredentialProvider(); expect(creds).toEqual({ accessKeyId: 'ACCESS_KEY_ID_10', secretAccessKey: 'SECRET_ACCESS_KEY_10', @@ -335,14 +335,14 @@ describe('DefaultAwsCredentialsProvider', () => { it('retrieves default cred provider chain from the main account when there is no AWS integration config', async () => { const minConfig = new ConfigReader({}); - const provider = DefaultAwsCredentialsProvider.fromConfig(minConfig); - const awsCredentials = await provider.getCredentials({ + const provider = DefaultAwsCredentialsManager.fromConfig(minConfig); + const awsCredentialProvider = await provider.getCredentialProvider({ accountId: '123456789012', }); - expect(awsCredentials.accountId).toEqual('123456789012'); + expect(awsCredentialProvider.accountId).toEqual('123456789012'); - const creds = await awsCredentials.provider(); + const creds = await awsCredentialProvider.sdkCredentialProvider(); expect(creds).toEqual({ accessKeyId: 'ACCESS_KEY_ID_10', secretAccessKey: 'SECRET_ACCESS_KEY_10', @@ -352,14 +352,14 @@ describe('DefaultAwsCredentialsProvider', () => { }); it('extracts the account ID from an ARN', async () => { - const provider = DefaultAwsCredentialsProvider.fromConfig(config); - const awsCredentials = await provider.getCredentials({ + const provider = DefaultAwsCredentialsManager.fromConfig(config); + const awsCredentialProvider = await provider.getCredentialProvider({ arn: 'arn:aws:ecs:region:111111111111:service/cluster-name/service-name', }); - expect(awsCredentials.accountId).toEqual('111111111111'); + expect(awsCredentialProvider.accountId).toEqual('111111111111'); - const creds = await awsCredentials.provider(); + const creds = await awsCredentialProvider.sdkCredentialProvider(); expect(creds).toEqual({ accessKeyId: 'ACCESS_KEY_ID_1', secretAccessKey: 'SECRET_ACCESS_KEY_1', @@ -369,14 +369,14 @@ describe('DefaultAwsCredentialsProvider', () => { }); it('falls back to main account credentials when account ID cannot be extracted from the ARN', async () => { - const provider = DefaultAwsCredentialsProvider.fromConfig(config); - const awsCredentials = await provider.getCredentials({ + const provider = DefaultAwsCredentialsManager.fromConfig(config); + const awsCredentialProvider = await provider.getCredentialProvider({ arn: 'arn:aws:s3:::bucket_name', }); - expect(awsCredentials.accountId).toEqual('123456789012'); + expect(awsCredentialProvider.accountId).toEqual('123456789012'); - const creds = await awsCredentials.provider(); + const creds = await awsCredentialProvider.sdkCredentialProvider(); expect(creds).toEqual({ accessKeyId: 'GHI', secretAccessKey: 'JKL', @@ -384,12 +384,12 @@ describe('DefaultAwsCredentialsProvider', () => { }); it('falls back to main account credentials when neither account ID nor ARN are provided', async () => { - const provider = DefaultAwsCredentialsProvider.fromConfig(config); - const awsCredentials = await provider.getCredentials({}); + const provider = DefaultAwsCredentialsManager.fromConfig(config); + const awsCredentialProvider = await provider.getCredentialProvider({}); - expect(awsCredentials.accountId).toEqual('123456789012'); + expect(awsCredentialProvider.accountId).toEqual('123456789012'); - const creds = await awsCredentials.provider(); + const creds = await awsCredentialProvider.sdkCredentialProvider(); expect(creds).toEqual({ accessKeyId: 'GHI', secretAccessKey: 'JKL', @@ -397,12 +397,12 @@ describe('DefaultAwsCredentialsProvider', () => { }); it('falls back to main account credentials when no options are provided', async () => { - const provider = DefaultAwsCredentialsProvider.fromConfig(config); - const awsCredentials = await provider.getCredentials(); + const provider = DefaultAwsCredentialsManager.fromConfig(config); + const awsCredentialProvider = await provider.getCredentialProvider(); - expect(awsCredentials.accountId).toEqual('123456789012'); + expect(awsCredentialProvider.accountId).toEqual('123456789012'); - const creds = await awsCredentials.provider(); + const creds = await awsCredentialProvider.sdkCredentialProvider(); expect(creds).toEqual({ accessKeyId: 'GHI', secretAccessKey: 'JKL', @@ -413,16 +413,16 @@ describe('DefaultAwsCredentialsProvider', () => { const minConfig = new ConfigReader({ aws: {}, }); - const provider = DefaultAwsCredentialsProvider.fromConfig(minConfig); + const provider = DefaultAwsCredentialsManager.fromConfig(minConfig); await expect( - provider.getCredentials({ accountId: '111222333444' }), + provider.getCredentialProvider({ accountId: '111222333444' }), ).rejects.toThrow(/no AWS integration that matches 111222333444/); }); it('rejects main account that has invalid credentials', async () => { stsMock.on(GetCallerIdentityCommand).rejects('No credentials found'); - const provider = DefaultAwsCredentialsProvider.fromConfig(config); - await expect(provider.getCredentials({})).rejects.toThrow( + const provider = DefaultAwsCredentialsManager.fromConfig(config); + await expect(provider.getCredentialProvider({})).rejects.toThrow( /No credentials found/, ); }); diff --git a/packages/integration-aws-node/src/DefaultAwsCredentialsProvider.ts b/packages/integration-aws-node/src/DefaultAwsCredentialsManager.ts similarity index 66% rename from packages/integration-aws-node/src/DefaultAwsCredentialsProvider.ts rename to packages/integration-aws-node/src/DefaultAwsCredentialsManager.ts index 8b26b5fe84..7799f08b4d 100644 --- a/packages/integration-aws-node/src/DefaultAwsCredentialsProvider.ts +++ b/packages/integration-aws-node/src/DefaultAwsCredentialsManager.ts @@ -21,9 +21,9 @@ import { AwsIntegrationMainAccountConfig, } from './config'; import { - AwsCredentials, - AwsCredentialsProvider, - AwsCredentialsProviderOptions, + AwsCredentialsManager, + AwsCredentialProvider, + AwsCredentialProviderOptions, } from './types'; import { GetCallerIdentityCommand, STSClient } from '@aws-sdk/client-sts'; import { @@ -36,20 +36,20 @@ import { parse } from '@aws-sdk/util-arn-parser'; import { Config } from '@backstage/config'; /** - * Retrieves the account ID for the given credentials provider from STS. + * Retrieves the account ID for the given credential provider from STS. */ -async function fillInAccountId(creds: AwsCredentials) { - if (creds.accountId) { +async function fillInAccountId(credProvider: AwsCredentialProvider) { + if (credProvider.accountId) { return; } const client = new STSClient({ - region: creds.stsRegion, - customUserAgent: 'backstage-aws-credentials-provider', - credentialDefaultProvider: () => creds.provider, + region: credProvider.stsRegion, + customUserAgent: 'backstage-aws-credentials-manager', + credentialDefaultProvider: () => credProvider.sdkCredentialProvider, }); const resp = await client.send(new GetCallerIdentityCommand({})); - creds.accountId = resp.Account!; + credProvider.accountId = resp.Account!; } function getStaticCredentials( @@ -72,7 +72,7 @@ function getProfileCredentials( profile, clientConfig: { region, - customUserAgent: 'backstage-aws-credentials-provider', + customUserAgent: 'backstage-aws-credentials-manager', }, }); } @@ -91,9 +91,9 @@ function getDefaultCredentialsChain(): AwsCredentialIdentityProvider { * 4. Profile creds * 5. Default AWS SDK creds chain */ -function getAccountCredentialsProvider( +function getSdkCredentialProvider( config: AwsIntegrationAccountConfig, - mainAccountCreds: AwsCredentialIdentityProvider, + mainAccountCredProvider: AwsCredentialIdentityProvider, ): AwsCredentialIdentityProvider { if (config.roleName) { const region = config.region ?? 'us-east-1'; @@ -102,7 +102,7 @@ function getAccountCredentialsProvider( return fromTemporaryCredentials({ masterCredentials: config.accessKeyId ? getStaticCredentials(config.accessKeyId!, config.secretAccessKey!) - : mainAccountCreds, + : mainAccountCredProvider, params: { RoleArn: `arn:${partition}:iam::${config.accountId}:role/${config.roleName}`, RoleSessionName: 'backstage', @@ -110,7 +110,7 @@ function getAccountCredentialsProvider( }, clientConfig: { region, - customUserAgent: 'backstage-aws-credentials-provider', + customUserAgent: 'backstage-aws-credentials-manager', }, }); } @@ -134,7 +134,7 @@ function getAccountCredentialsProvider( * 2. Profile creds * 3. Default AWS SDK creds chain */ -function getMainAccountCredentialsProvider( +function getMainAccountSdkCredentialProvider( config: AwsIntegrationMainAccountConfig, ): AwsCredentialIdentityProvider { if (config.accessKeyId) { @@ -153,8 +153,8 @@ function getMainAccountCredentialsProvider( * * @public */ -export class DefaultAwsCredentialsProvider implements AwsCredentialsProvider { - static fromConfig(config: Config): DefaultAwsCredentialsProvider { +export class DefaultAwsCredentialsManager implements AwsCredentialsManager { + static fromConfig(config: Config): DefaultAwsCredentialsManager { const awsConfig = config.has('aws') ? readAwsIntegrationConfig(config.getConfig('aws')) : { @@ -163,63 +163,66 @@ export class DefaultAwsCredentialsProvider implements AwsCredentialsProvider { accountDefaults: {}, }; - const mainAccountProvider = getMainAccountCredentialsProvider( + const mainAccountSdkCredProvider = getMainAccountSdkCredentialProvider( awsConfig.mainAccount, ); - const mainAccountCreds: AwsCredentials = { - provider: mainAccountProvider, + const mainAccountCredProvider: AwsCredentialProvider = { + sdkCredentialProvider: mainAccountSdkCredProvider, }; - const accountCreds = new Map(); + const accountCredProviders = new Map(); for (const accountConfig of awsConfig.accounts) { - const provider = getAccountCredentialsProvider( + const sdkCredentialProvider = getSdkCredentialProvider( accountConfig, - mainAccountCreds.provider, + mainAccountSdkCredProvider, ); - accountCreds.set(accountConfig.accountId, { + accountCredProviders.set(accountConfig.accountId, { accountId: accountConfig.accountId, stsRegion: accountConfig.region, - provider, + sdkCredentialProvider, }); } - return new DefaultAwsCredentialsProvider( - accountCreds, + return new DefaultAwsCredentialsManager( + accountCredProviders, awsConfig.accountDefaults, - mainAccountCreds, + mainAccountCredProvider, ); } private constructor( - private readonly accountCredentials: Map, + private readonly accountCredentialProviders: Map< + string, + AwsCredentialProvider + >, private readonly accountDefaults: AwsIntegrationDefaultAccountConfig, - private readonly mainAccountCredentials: AwsCredentials, + private readonly mainAccountCredentialProvider: AwsCredentialProvider, ) {} /** - * Returns {@link AwsCredentials} for a given AWS account. + * Returns an {@link AwsCredentialProvider} for a given AWS account. * * @example * ```ts - * const { provider } = await getCredentials({ + * const { provider } = await getCredentialProvider({ * accountId: '0123456789012', * }) * - * const { provider } = await getCredentials({ + * const { provider } = await getCredentialProvider({ * arn: 'arn:aws:ecs:us-west-2:123456789012:service/my-http-service' * }) * ``` * * @param opts - the AWS account ID or AWS resource ARN - * @returns A promise of {@link AwsCredentials}. + * @returns A promise of {@link AwsCredentialProvider}. */ - async getCredentials( - opts?: AwsCredentialsProviderOptions, - ): Promise { + async getCredentialProvider( + opts?: AwsCredentialProviderOptions, + ): Promise { // If no options provided, fall back to the main account if (!opts) { - await fillInAccountId(this.mainAccountCredentials); - return this.mainAccountCredentials; + await fillInAccountId(this.mainAccountCredentialProvider); + return this.mainAccountCredentialProvider; } // Determine the account ID: either explicitly provided or extracted from the provided ARN @@ -232,13 +235,13 @@ export class DefaultAwsCredentialsProvider implements AwsCredentialsProvider { // If the account ID was not provided (explicitly or in the ARN), // fall back to the main account if (!accountId) { - await fillInAccountId(this.mainAccountCredentials); - return this.mainAccountCredentials; + await fillInAccountId(this.mainAccountCredentialProvider); + return this.mainAccountCredentialProvider; } // Return a cached provider if available - if (this.accountCredentials.has(accountId)) { - return this.accountCredentials.get(accountId)!; + if (this.accountCredentialProviders.has(accountId)) { + return this.accountCredentialProviders.get(accountId)!; } // First, fall back to using the account defaults @@ -250,20 +253,23 @@ export class DefaultAwsCredentialsProvider implements AwsCredentialsProvider { region: this.accountDefaults.region, externalId: this.accountDefaults.externalId, }; - const provider = getAccountCredentialsProvider( + const sdkCredentialProvider = getSdkCredentialProvider( config, - this.mainAccountCredentials.provider, + this.mainAccountCredentialProvider.sdkCredentialProvider, ); - const creds: AwsCredentials = { accountId, provider }; - this.accountCredentials.set(accountId, creds); - return creds; + const credProvider: AwsCredentialProvider = { + accountId, + sdkCredentialProvider, + }; + this.accountCredentialProviders.set(accountId, credProvider); + return credProvider; } // Then, fall back to using the main account, but only // if the account requested matches the main account ID - await fillInAccountId(this.mainAccountCredentials); - if (accountId === this.mainAccountCredentials.accountId) { - return this.mainAccountCredentials; + await fillInAccountId(this.mainAccountCredentialProvider); + if (accountId === this.mainAccountCredentialProvider.accountId) { + return this.mainAccountCredentialProvider; } // Otherwise, the account needs to be explicitly configured in Backstage diff --git a/packages/integration-aws-node/src/index.ts b/packages/integration-aws-node/src/index.ts index d0a86acb47..0b6dc630db 100644 --- a/packages/integration-aws-node/src/index.ts +++ b/packages/integration-aws-node/src/index.ts @@ -21,9 +21,9 @@ export type { AwsIntegrationDefaultAccountConfig, AwsIntegrationMainAccountConfig, } from './config'; -export { DefaultAwsCredentialsProvider } from './DefaultAwsCredentialsProvider'; +export { DefaultAwsCredentialsManager } from './DefaultAwsCredentialsManager'; export type { - AwsCredentials, - AwsCredentialsProvider, - AwsCredentialsProviderOptions, + AwsCredentialsManager, + AwsCredentialProvider, + AwsCredentialProviderOptions, } from './types'; diff --git a/packages/integration-aws-node/src/types.ts b/packages/integration-aws-node/src/types.ts index 44f7850cf8..3ff069b20b 100644 --- a/packages/integration-aws-node/src/types.ts +++ b/packages/integration-aws-node/src/types.ts @@ -20,10 +20,19 @@ import { AwsCredentialIdentityProvider } from '@aws-sdk/types'; * * @public */ -export type AwsCredentials = { +export type AwsCredentialProvider = { + /** + * The AWS account ID of these credentials + */ accountId?: string; + /** + * The STS region used with these credentials + */ stsRegion?: string; - provider: AwsCredentialIdentityProvider; + /** + * The credential identity provider to use when creating AWS SDK for Javascript V3 clients + */ + sdkCredentialProvider: AwsCredentialIdentityProvider; }; /** @@ -31,7 +40,7 @@ export type AwsCredentials = { * * @public */ -export type AwsCredentialsProviderOptions = { +export type AwsCredentialProviderOptions = { /** * The AWS account ID, e.g. '0123456789012' */ @@ -49,9 +58,11 @@ export type AwsCredentialsProviderOptions = { * * @public */ -export interface AwsCredentialsProvider { +export interface AwsCredentialsManager { /** * Get credentials for an AWS account. */ - getCredentials(opts?: AwsCredentialsProviderOptions): Promise; + getCredentialProvider( + opts?: AwsCredentialProviderOptions, + ): Promise; } diff --git a/plugins/techdocs-node/src/stages/publish/awsS3.test.ts b/plugins/techdocs-node/src/stages/publish/awsS3.test.ts index ff61ac58a2..f3b2162a6b 100644 --- a/plugins/techdocs-node/src/stages/publish/awsS3.test.ts +++ b/plugins/techdocs-node/src/stages/publish/awsS3.test.ts @@ -28,9 +28,9 @@ import { getVoidLogger } from '@backstage/backend-common'; import { Entity, DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { - AwsCredentials, - AwsCredentialsProviderOptions, - DefaultAwsCredentialsProvider, + AwsCredentialProvider, + AwsCredentialProviderOptions, + DefaultAwsCredentialsManager, } from '@backstage/integration-aws-node'; import { mockClient, AwsClientStub } from 'aws-sdk-client-mock'; import express from 'express'; @@ -45,9 +45,9 @@ import { Readable } from 'stream'; const env = process.env; let s3Mock: AwsClientStub; -function getMockCredentials(): Promise { +function getMockCredentialProvider(): Promise { return Promise.resolve({ - provider: async () => { + sdkCredentialProvider: async () => { return Promise.resolve({ accessKeyId: 'MY_ACCESS_KEY_ID', secretAccessKey: 'MY_SECRET_ACCESS_KEY', @@ -55,9 +55,9 @@ function getMockCredentials(): Promise { }, }); } -const credsProviderMock = jest.spyOn( - DefaultAwsCredentialsProvider.prototype, - 'getCredentials', +const getCredProviderMock = jest.spyOn( + DefaultAwsCredentialsManager.prototype, + 'getCredentialProvider', ); const getEntityRootDir = (entity: Entity) => { @@ -178,8 +178,8 @@ describe('AwsS3Publish', () => { process.env.AWS_REGION = 'us-west-2'; jest.resetAllMocks(); - credsProviderMock.mockImplementation((_?: AwsCredentialsProviderOptions) => - getMockCredentials(), + getCredProviderMock.mockImplementation((_?: AwsCredentialProviderOptions) => + getMockCredentialProvider(), ); mockFs({ @@ -249,10 +249,10 @@ describe('AwsS3Publish', () => { describe('buildCredentials', () => { it('should retrieve credentials for a specific account ID', async () => { await createPublisherFromConfig(); - expect(credsProviderMock).toHaveBeenCalledWith({ + expect(getCredProviderMock).toHaveBeenCalledWith({ accountId: '111111111111', }); - expect(credsProviderMock).toHaveBeenCalledTimes(1); + expect(getCredProviderMock).toHaveBeenCalledTimes(1); }); it('should retrieve default credentials when no config is present', async () => { @@ -268,8 +268,8 @@ describe('AwsS3Publish', () => { }); await AwsS3Publish.fromConfig(mockConfig, logger); - expect(credsProviderMock).toHaveBeenCalledWith(); - expect(credsProviderMock).toHaveBeenCalledTimes(1); + expect(getCredProviderMock).toHaveBeenCalledWith(); + expect(getCredProviderMock).toHaveBeenCalledTimes(1); }); it('should fall back to deprecated method of retrieving credentials', async () => { @@ -290,7 +290,7 @@ describe('AwsS3Publish', () => { }); await AwsS3Publish.fromConfig(mockConfig, logger); - expect(credsProviderMock).toHaveBeenCalledTimes(0); + expect(getCredProviderMock).toHaveBeenCalledTimes(0); }); }); diff --git a/plugins/techdocs-node/src/stages/publish/awsS3.ts b/plugins/techdocs-node/src/stages/publish/awsS3.ts index c71a25cfe4..a13baef926 100644 --- a/plugins/techdocs-node/src/stages/publish/awsS3.ts +++ b/plugins/techdocs-node/src/stages/publish/awsS3.ts @@ -17,8 +17,8 @@ import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError, ForwardedError } from '@backstage/errors'; import { - AwsCredentialsProvider, - DefaultAwsCredentialsProvider, + AwsCredentialsManager, + DefaultAwsCredentialsManager, } from '@backstage/integration-aws-node'; import { GetObjectCommand, @@ -136,9 +136,9 @@ export class AwsS3Publish implements PublisherBase { const credentialsConfig = config.getOptionalConfig( 'techdocs.publisher.awsS3.credentials', ); - const credsProvider = DefaultAwsCredentialsProvider.fromConfig(config); - const credentials = await AwsS3Publish.buildCredentials( - credsProvider, + const credsManager = DefaultAwsCredentialsManager.fromConfig(config); + const sdkCredentialProvider = await AwsS3Publish.buildCredentials( + credsManager, accountId, credentialsConfig, region, @@ -158,7 +158,7 @@ export class AwsS3Publish implements PublisherBase { const storageClient = new S3Client({ customUserAgent: 'backstage-aws-techdocs-s3-publisher', - credentialDefaultProvider: () => credentials, + credentialDefaultProvider: () => sdkCredentialProvider, ...(region && { region }), ...(endpoint && { endpoint }), ...(s3ForcePathStyle && { s3ForcePathStyle }), @@ -192,20 +192,21 @@ export class AwsS3Publish implements PublisherBase { } private static async buildCredentials( - credsProvider: AwsCredentialsProvider, + credsManager: AwsCredentialsManager, accountId?: string, config?: Config, region?: string, ): Promise { // Pull credentials for the specified account ID from the 'aws' config section if (accountId) { - return (await credsProvider.getCredentials({ accountId })).provider; + return (await credsManager.getCredentialProvider({ accountId })) + .sdkCredentialProvider; } // Fall back to the default credential chain if neither account ID // nor explicit credentials are provided if (!config) { - return (await credsProvider.getCredentials()).provider; + return (await credsManager.getCredentialProvider()).sdkCredentialProvider; } // Pull credentials from the techdocs config section (deprecated) @@ -214,7 +215,7 @@ export class AwsS3Publish implements PublisherBase { const explicitCredentials: AwsCredentialIdentityProvider = accessKeyId && secretAccessKey ? AwsS3Publish.buildStaticCredentials(accessKeyId, secretAccessKey) - : (await credsProvider.getCredentials()).provider; + : (await credsManager.getCredentialProvider()).sdkCredentialProvider; const roleArn = config.getOptionalString('roleArn'); if (roleArn) { From 35af08d9a03afea0584a951e9d4395ff462315e0 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 29 Nov 2022 15:14:30 +0100 Subject: [PATCH 055/437] Update elastic engine to preserve pre-existing indices if indexer receives 0 documents Signed-off-by: Eric Peterson --- .../src/engines/ElasticSearchSearchEngine.ts | 11 ++++++---- .../ElasticSearchSearchEngineIndexer.test.ts | 20 +++++++++++++++++++ .../ElasticSearchSearchEngineIndexer.ts | 18 +++++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index f3db46956d..f17ec1b01d 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -252,6 +252,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { async getIndexer(type: string) { const alias = this.constructSearchAlias(type); + const indexerLogger = this.logger.child({ documentType: type }); const indexer = new ElasticSearchSearchEngineIndexer({ type, @@ -259,13 +260,13 @@ export class ElasticSearchSearchEngine implements SearchEngine { indexSeparator: this.indexSeparator, alias, elasticSearchClientWrapper: this.elasticSearchClientWrapper, - logger: this.logger, + logger: indexerLogger, batchSize: this.batchSize, }); // Attempt cleanup upon failure. indexer.on('error', async e => { - this.logger.error(`Failed to index documents for type ${type}`, e); + indexerLogger.error(`Failed to index documents for type ${type}`, e); let cleanupError: Error | undefined; // In some cases, a failure may have occurred before the indexer was able @@ -296,11 +297,13 @@ export class ElasticSearchSearchEngine implements SearchEngine { }); if (cleanupError) { - this.logger.error( + indexerLogger.error( `Unable to clean up elastic index ${indexer.indexName}: ${cleanupError}`, ); } else { - this.logger.info(`Removed partial, failed index ${indexer.indexName}`); + indexerLogger.info( + `Removed partial, failed index ${indexer.indexName}`, + ); } }); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts index 155a6bbf32..1e9e98dbaf 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -176,6 +176,26 @@ describe('ElasticSearchSearchEngineIndexer', () => { expect(deleteSpy).toHaveBeenCalled(); }); + it('handles when no documents are received', async () => { + await TestPipeline.fromIndexer(indexer).withDocuments([]).execute(); + + // Older indices should have been queried for. + expect(catSpy).toHaveBeenCalled(); + + // A new index should have been created. + const createdIndex = createSpy.mock.calls[0][0].path.slice(1); + expect(createdIndex).toContain('some-type-index__'); + + // No documents should have been sent + expect(bulkSpy).not.toHaveBeenCalled(); + + // Alias should not have been rotated. + expect(aliasesSpy).not.toHaveBeenCalled(); + + // Old index should not be cleaned up. + expect(deleteSpy).not.toHaveBeenCalled(); + }); + it('handles bulk and batching during indexing', async () => { const documents = range(550).map(i => ({ title: `Hello World ${i}`, diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index e32c49559d..1b1d47bb9b 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -131,6 +131,24 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { // Wait for the bulk helper to finish processing. const result = await this.bulkResult; + // Warn that no documents were indexed, early return so that alias swapping + // does not occur, and clean up the empty index we just created. + if (this.processed === 0) { + this.logger.warn( + `Index for ${this.type} was not ${ + this.removableIndices.length ? 'replaced' : 'created' + }: indexer received 0 documents`, + ); + try { + await this.elasticSearchClientWrapper.deleteIndex({ + index: this.indexName, + }); + } catch (error) { + this.logger.error(`Unable to clean up elastic index: ${error}`); + } + return; + } + // Rotate main alias upon completion. Apply permanent secondary alias so // stale indices can be referenced for deletion in case initial attempt // fails. Allow errors to bubble up so that we can clean up the created index. From dff98437181c229b20f87f3c457d89ab37d22db5 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 29 Nov 2022 15:20:10 +0100 Subject: [PATCH 056/437] Changeset describing updated search indexer behavior Signed-off-by: Eric Peterson --- .changeset/search-with-alcohol.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/search-with-alcohol.md diff --git a/.changeset/search-with-alcohol.md b/.changeset/search-with-alcohol.md new file mode 100644 index 0000000000..a46f33985a --- /dev/null +++ b/.changeset/search-with-alcohol.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': minor +'@backstage/plugin-search-backend-module-pg': minor +'@backstage/plugin-search-backend-node': minor +--- + +The search engine now better handles the case when it receives 0 documents at index-time. Prior to this change, the indexer would replace any existing index with an empty index, effectively deleting it. Now instead, a warning is logged, and any existing index is left alone (preserving the index from the last successful indexing attempt). From d75d73d33bfb2d21f8e8cea10d35c98dc39fea50 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 2 Dec 2022 11:46:47 +0100 Subject: [PATCH 057/437] Clearer test assertions Co-authored-by: Renan Mendes Carvalho Signed-off-by: Eric Peterson --- .../src/engines/ElasticSearchSearchEngineIndexer.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts index 1e9e98dbaf..92193976b4 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -183,8 +183,12 @@ describe('ElasticSearchSearchEngineIndexer', () => { expect(catSpy).toHaveBeenCalled(); // A new index should have been created. - const createdIndex = createSpy.mock.calls[0][0].path.slice(1); - expect(createdIndex).toContain('some-type-index__'); + expect(createSpy).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + path: expect.stringContaining('some-type-index__'), + }), + ); // No documents should have been sent expect(bulkSpy).not.toHaveBeenCalled(); From 3dc6a522e96e724274e845b25759c475c54ae9a0 Mon Sep 17 00:00:00 2001 From: Juan Pablo Garcia Ripa Date: Fri, 2 Dec 2022 15:21:24 +0100 Subject: [PATCH 058/437] move `resolvePackagePath` to lib/path + tests Signed-off-by: Juan Pablo Garcia Ripa --- packages/repo-tools/package.json | 5 +- .../src/commands/api-reports/api-extractor.ts | 21 +----- packages/repo-tools/src/lib/paths.test.ts | 66 +++++++++++++++++++ packages/repo-tools/src/lib/paths.ts | 23 +++++++ yarn.lock | 3 + 5 files changed, 97 insertions(+), 21 deletions(-) create mode 100644 packages/repo-tools/src/lib/paths.test.ts diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index 657bbb4404..9b51d35922 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -44,7 +44,10 @@ "ts-node": "^10.0.0" }, "devDependencies": { - "@types/is-glob": "^4.0.2" + "@backstage/cli": "workspace:^", + "@types/is-glob": "^4.0.2", + "@types/mock-fs": "^4.13.0", + "mock-fs": "^5.1.0" }, "files": [ "bin", diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index d1e8d86240..b2fab40ca5 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -65,7 +65,7 @@ import { } from '@microsoft/api-documenter/lib/markdown/CustomMarkdownEmitter'; import { IMarkdownEmitterContext } from '@microsoft/api-documenter/lib/markdown/MarkdownEmitter'; import { AstDeclaration } from '@microsoft/api-extractor/lib/analyzer/AstDeclaration'; -import { paths as cliPaths } from '../../lib/paths'; +import { paths as cliPaths, resolvePackagePath } from '../../lib/paths'; import g from 'glob'; import isGlob from 'is-glob'; @@ -226,25 +226,6 @@ ApiReportGenerator.generateReviewFileContent = }); }; -export async function resolvePackagePath( - packagePath: string, -): Promise { - const fullPackageDir = cliPaths.resolveTargetRoot(packagePath); - - const stat = await fs.stat(fullPackageDir); - if (!stat.isDirectory()) { - return undefined; - } - - try { - const packageJsonPath = join(fullPackageDir, 'package.json'); - await fs.access(packageJsonPath); - } catch (_) { - return undefined; - } - return relativePath(cliPaths.targetRoot, fullPackageDir); -} - export async function findPackageDirs(selectedPaths: string[]) { const packageDirs = new Array(); for (const packageRoot of selectedPaths) { diff --git a/packages/repo-tools/src/lib/paths.test.ts b/packages/repo-tools/src/lib/paths.test.ts new file mode 100644 index 0000000000..90d84aec3c --- /dev/null +++ b/packages/repo-tools/src/lib/paths.test.ts @@ -0,0 +1,66 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import mockFs from 'mock-fs'; +import { resolve as resolvePath } from 'path'; +import { resolvePackagePath, paths } from './paths'; + +describe('paths', () => { + jest.spyOn(paths, 'targetRoot', 'get').mockReturnValue('/root'); + jest.spyOn(paths, 'resolveTargetRoot').mockImplementation((...path) => { + return resolvePath('/root', ...path); + }); + + beforeEach(() => { + mockFs({ + [paths.targetRoot]: { + 'package.json': JSON.stringify({ name: 'test' }), + packages: { + 'package-a': { + 'package.json': '{}', + }, + 'package-b': { + 'package.json': '{}', + }, + 'package-c': {}, + 'README.md': 'Hello World', + }, + }, + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + describe('resolvePackagePath', () => { + it('should return undefined if the package does not exist or does not contain a package.json', async () => { + expect(await resolvePackagePath('packages/package-d')).toBeUndefined(); + expect(await resolvePackagePath('packages/package-c')).toBeUndefined(); + }); + it('should return the path to the package if it exists and has a package.json', async () => { + expect(await resolvePackagePath('packages/package-a')).toBe( + 'packages/package-a', + ); + expect(await resolvePackagePath('packages/package-b')).toBe( + 'packages/package-b', + ); + }); + it('should return undefined if the pat is not a directory', async () => { + expect(await resolvePackagePath('packages/README.md')).toBeUndefined(); + }); + }); +}); diff --git a/packages/repo-tools/src/lib/paths.ts b/packages/repo-tools/src/lib/paths.ts index 2c658c27b3..f387e9be51 100644 --- a/packages/repo-tools/src/lib/paths.ts +++ b/packages/repo-tools/src/lib/paths.ts @@ -15,6 +15,29 @@ */ import { findPaths } from '@backstage/cli-common'; +import { relative as relativePath, join } from 'path'; +import fs from 'fs-extra'; /* eslint-disable-next-line no-restricted-syntax */ export const paths = findPaths(__dirname); + +export async function resolvePackagePath( + packagePath: string, +): Promise { + const fullPackageDir = paths.resolveTargetRoot(packagePath); + + try { + const stat = await fs.stat(fullPackageDir); + if (!stat.isDirectory()) { + return undefined; + } + + const packageJsonPath = join(fullPackageDir, 'package.json'); + + await fs.access(packageJsonPath); + } catch (e) { + console.log(`folder omitted: ${fullPackageDir}, cause: ${e}`); + return undefined; + } + return relativePath(paths.targetRoot, fullPackageDir); +} diff --git a/yarn.lock b/yarn.lock index c9a9d1daea..e92df381ca 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8467,6 +8467,7 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/repo-tools@workspace:packages/repo-tools" dependencies: + "@backstage/cli": "workspace:^" "@backstage/cli-common": "workspace:^" "@backstage/errors": "workspace:^" "@manypkg/get-packages": ^1.1.3 @@ -8475,10 +8476,12 @@ __metadata: "@microsoft/api-extractor-model": ^7.17.2 "@microsoft/tsdoc": 0.14.1 "@types/is-glob": ^4.0.2 + "@types/mock-fs": ^4.13.0 chalk: ^4.0.0 commander: ^9.1.0 fs-extra: 10.1.0 is-glob: ^4.0.3 + mock-fs: ^5.1.0 ts-node: ^10.0.0 bin: backstage-repo-tools: bin/backstage-repo-tools From b5e3da44c4155fd2b074696a05dc8c2a492f668b Mon Sep 17 00:00:00 2001 From: TheMonolithX64 Date: Fri, 2 Dec 2022 17:42:20 +0000 Subject: [PATCH 059/437] Update broken links to cli docs Signed-off-by: TheMonolithX64 --- docs/getting-started/keeping-backstage-updated.md | 4 ++-- packages/backend/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/getting-started/keeping-backstage-updated.md b/docs/getting-started/keeping-backstage-updated.md index 64f78c0997..8e130e7094 100644 --- a/docs/getting-started/keeping-backstage-updated.md +++ b/docs/getting-started/keeping-backstage-updated.md @@ -13,7 +13,7 @@ starting point that's meant to be evolved. The Backstage CLI has a command to bump all `@backstage` packages and dependencies you're using to the latest versions: -[versions:bump](https://backstage.io/docs/cli/commands#versionsbump). +[versions:bump](https://backstage.io/docs/local-dev/cli-commands#versionsbump). ```bash yarn backstage-cli versions:bump @@ -70,7 +70,7 @@ example, depends on global referential equality. This can cause problems in Backstage with API lookup, or config loading. To help resolve these situations, the Backstage CLI has -[versions:check](https://backstage.io/docs/cli/commands#versionscheck). This +[versions:check](https://backstage.io/docs/local-dev/cli-commands#versionscheck). This will validate versions of `@backstage` packages in your app to check for duplicate definitions: diff --git a/packages/backend/README.md b/packages/backend/README.md index f4c115db65..e4fd81749d 100644 --- a/packages/backend/README.md +++ b/packages/backend/README.md @@ -31,7 +31,7 @@ The backend starts up on port 7007 per default. ### Debugging -The backend is a node process that can be inspected to allow breakpoints and live debugging. To enable this, pass the `--inspect` flag to [backend:dev](https://backstage.io/docs/cli/commands#backenddev). +The backend is a node process that can be inspected to allow breakpoints and live debugging. To enable this, pass the `--inspect` flag to [backend:dev](https://backstage.io/docs/local-dev/cli-build-system#backend-development). To debug the backend in [Visual Studio Code](https://code.visualstudio.com/): From ea4192ff682c26fd55b8a8b958e46408821103e5 Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Fri, 2 Dec 2022 16:05:20 -0500 Subject: [PATCH 060/437] add more type declarations Signed-off-by: Jamie Klassen --- .../src/service/KubernetesClientProvider.ts | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts b/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts index 522925beb5..70764fed66 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts @@ -15,10 +15,13 @@ */ import { + Cluster, + Context, CoreV1Api, + CustomObjectsApi, KubeConfig, Metrics, - CustomObjectsApi, + User, } from '@kubernetes/client-node'; import { ClusterDetails } from '../types/types'; @@ -29,26 +32,26 @@ import { ClusterDetails } from '../types/types'; export class KubernetesClientProvider { // visible for testing getKubeConfig(clusterDetails: ClusterDetails) { - const cluster = { + const cluster: Cluster = { name: clusterDetails.name, server: clusterDetails.url, - skipTLSVerify: clusterDetails.skipTLSVerify, + skipTLSVerify: clusterDetails.skipTLSVerify || false, caData: clusterDetails.caData, }; // TODO configure - const user = { + const user: User = { name: 'backstage', token: clusterDetails.serviceAccountToken, }; - const context = { + const context: Context = { name: `${clusterDetails.name}`, user: user.name, cluster: cluster.name, }; - const kc = new KubeConfig(); + const kc: KubeConfig = new KubeConfig(); if (clusterDetails.serviceAccountToken) { kc.loadFromOptions({ clusters: [cluster], @@ -63,19 +66,19 @@ export class KubernetesClientProvider { return kc; } - getCoreClientByClusterDetails(clusterDetails: ClusterDetails) { + getCoreClientByClusterDetails(clusterDetails: ClusterDetails): CoreV1Api { const kc = this.getKubeConfig(clusterDetails); return kc.makeApiClient(CoreV1Api); } - getMetricsClient(clusterDetails: ClusterDetails) { + getMetricsClient(clusterDetails: ClusterDetails): Metrics { const kc = this.getKubeConfig(clusterDetails); return new Metrics(kc); } - getCustomObjectsClient(clusterDetails: ClusterDetails) { + getCustomObjectsClient(clusterDetails: ClusterDetails): CustomObjectsApi { const kc = this.getKubeConfig(clusterDetails); return kc.makeApiClient(CustomObjectsApi); From 22e20b3a5966f6f0b34ee3a493b37d60af441784 Mon Sep 17 00:00:00 2001 From: Jamie Klassen Date: Fri, 2 Dec 2022 18:23:18 -0500 Subject: [PATCH 061/437] clusters in app-config support caFile Signed-off-by: Jamie Klassen --- .changeset/kind-tips-pump.md | 5 ++ docs/features/kubernetes/configuration.md | 13 ++++- plugins/kubernetes-backend/api-report.md | 2 + plugins/kubernetes-backend/config.d.ts | 4 ++ plugins/kubernetes-backend/package.json | 1 + .../ConfigClusterLocator.test.ts | 8 +++ .../cluster-locator/ConfigClusterLocator.ts | 1 + .../src/cluster-locator/index.test.ts | 2 + .../service/KubernetesClientProvider.test.ts | 56 +++++++++++++------ .../src/service/KubernetesClientProvider.ts | 3 +- plugins/kubernetes-backend/src/types/types.ts | 1 + yarn.lock | 3 +- 12 files changed, 78 insertions(+), 21 deletions(-) create mode 100644 .changeset/kind-tips-pump.md diff --git a/.changeset/kind-tips-pump.md b/.changeset/kind-tips-pump.md new file mode 100644 index 0000000000..c61ed97774 --- /dev/null +++ b/.changeset/kind-tips-pump.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-kubernetes-backend': patch +--- + +Clusters declared in the app-config can now have their CA configured via a local filesystem path using the `caFile` property. diff --git a/docs/features/kubernetes/configuration.md b/docs/features/kubernetes/configuration.md index 55843f3b24..41d72b2be2 100644 --- a/docs/features/kubernetes/configuration.md +++ b/docs/features/kubernetes/configuration.md @@ -31,6 +31,7 @@ kubernetes: dashboardUrl: http://127.0.0.1:64713 # url copied from running the command: minikube service kubernetes-dashboard -n kubernetes-dashboard dashboardApp: standard caData: ${K8S_CONFIG_CA_DATA} + caFile: '' # local path to CA file customResources: - group: 'argoproj.io' apiVersion: 'v1alpha1' @@ -248,8 +249,8 @@ kubernetes: ##### `clusters.\*.caData` (optional) Base64-encoded certificate authority bundle in PEM format. The Kubernetes client -will verify that TLS certificate presented by the API server is signed by this -CA. +will verify that the TLS certificate presented by the API server is signed by +this CA. This value could be obtained via inspecting the kubeconfig file (usually at `~/.kube/config`) under `clusters[*].cluster.certificate-authority-data`. For @@ -265,6 +266,14 @@ See also https://cloud.google.com/kubernetes-engine/docs/how-to/api-server-authentication#environments-without-gcloud for complete docs about GKE without `gcloud`. +##### `clusters.\*.caFile` (optional) + +Filesystem path (on the host where the Backstage process is running) to a +certificate authority bundle in PEM format. The Kubernetes client will verify +that the TLS certificate presented by the API server is signed by this CA. Note +that only clusters defined in the app-config via the [`config`](#config) +cluster locator method can be configured in this way. + ##### `clusters.\*.customResources` (optional) Configures which [custom resources][3] to look for when returning an entity's diff --git a/plugins/kubernetes-backend/api-report.md b/plugins/kubernetes-backend/api-report.md index b88b3f2a02..2161df7c9a 100644 --- a/plugins/kubernetes-backend/api-report.md +++ b/plugins/kubernetes-backend/api-report.md @@ -78,6 +78,8 @@ export interface ClusterDetails { authProvider: string; // (undocumented) caData?: string | undefined; + // (undocumented) + caFile?: string | undefined; customResources?: CustomResourceMatcher[]; dashboardApp?: string; dashboardParameters?: JsonObject; diff --git a/plugins/kubernetes-backend/config.d.ts b/plugins/kubernetes-backend/config.d.ts index 2a514394d8..cc6c8bcd46 100644 --- a/plugins/kubernetes-backend/config.d.ts +++ b/plugins/kubernetes-backend/config.d.ts @@ -54,6 +54,10 @@ export interface Config { skipTLSVerify?: boolean; /** @visibility frontend */ skipMetricsLookup?: boolean; + /** @visibility secret */ + caData?: string; + /** @visibility secret */ + caFile?: string; }>; } | { diff --git a/plugins/kubernetes-backend/package.json b/plugins/kubernetes-backend/package.json index 100b67839f..4a54887cef 100644 --- a/plugins/kubernetes-backend/package.json +++ b/plugins/kubernetes-backend/package.json @@ -70,6 +70,7 @@ "@types/aws4": "^1.5.1", "@types/http-proxy-middleware": "^0.19.3", "aws-sdk-mock": "^5.2.1", + "mock-fs": "^5.2.0", "msw": "^0.49.0", "supertest": "^6.1.3" }, diff --git a/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.test.ts b/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.test.ts index 6ece380f28..5d8fefc5d2 100644 --- a/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.test.ts +++ b/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.test.ts @@ -55,6 +55,7 @@ describe('ConfigClusterLocator', () => { skipMetricsLookup: false, skipTLSVerify: false, caData: undefined, + caFile: undefined, }, ]); }); @@ -95,6 +96,7 @@ describe('ConfigClusterLocator', () => { skipTLSVerify: false, skipMetricsLookup: true, caData: undefined, + caFile: undefined, }, { name: 'cluster2', @@ -104,6 +106,7 @@ describe('ConfigClusterLocator', () => { skipTLSVerify: true, skipMetricsLookup: false, caData: undefined, + caFile: undefined, }, ]); }); @@ -151,6 +154,7 @@ describe('ConfigClusterLocator', () => { skipTLSVerify: false, skipMetricsLookup: false, caData: undefined, + caFile: undefined, }, { assumeRole: 'SomeRole', @@ -162,6 +166,7 @@ describe('ConfigClusterLocator', () => { skipTLSVerify: true, skipMetricsLookup: false, caData: undefined, + caFile: undefined, }, { assumeRole: 'SomeRole', @@ -173,6 +178,7 @@ describe('ConfigClusterLocator', () => { skipTLSVerify: true, skipMetricsLookup: false, caData: undefined, + caFile: undefined, }, ]); }); @@ -207,6 +213,7 @@ describe('ConfigClusterLocator', () => { skipMetricsLookup: false, skipTLSVerify: false, caData: undefined, + caFile: undefined, dashboardApp: 'gke', dashboardParameters: { projectId: 'some-project', @@ -243,6 +250,7 @@ describe('ConfigClusterLocator', () => { skipMetricsLookup: false, skipTLSVerify: false, caData: undefined, + caFile: undefined, dashboardApp: 'standard', dashboardUrl: 'http://someurl', }, diff --git a/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.ts b/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.ts index de5cac6d8d..b0143c2010 100644 --- a/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.ts +++ b/plugins/kubernetes-backend/src/cluster-locator/ConfigClusterLocator.ts @@ -37,6 +37,7 @@ export class ConfigClusterLocator implements KubernetesClustersSupplier { skipTLSVerify: c.getOptionalBoolean('skipTLSVerify') ?? false, skipMetricsLookup: c.getOptionalBoolean('skipMetricsLookup') ?? false, caData: c.getOptionalString('caData'), + caFile: c.getOptionalString('caFile'), authProvider: authProvider, }; const dashboardUrl = c.getOptionalString('dashboardUrl'); diff --git a/plugins/kubernetes-backend/src/cluster-locator/index.test.ts b/plugins/kubernetes-backend/src/cluster-locator/index.test.ts index a43e475d8e..6847b0e0a3 100644 --- a/plugins/kubernetes-backend/src/cluster-locator/index.test.ts +++ b/plugins/kubernetes-backend/src/cluster-locator/index.test.ts @@ -60,6 +60,7 @@ describe('getCombinedClusterSupplier', () => { skipMetricsLookup: false, skipTLSVerify: false, caData: undefined, + caFile: undefined, }, { name: 'cluster2', @@ -69,6 +70,7 @@ describe('getCombinedClusterSupplier', () => { skipMetricsLookup: false, skipTLSVerify: false, caData: undefined, + caFile: undefined, }, ]); }); diff --git a/plugins/kubernetes-backend/src/service/KubernetesClientProvider.test.ts b/plugins/kubernetes-backend/src/service/KubernetesClientProvider.test.ts index e813a63f0d..0f73d129b3 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesClientProvider.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesClientProvider.test.ts @@ -16,26 +16,29 @@ import '@backstage/backend-common'; import { KubernetesClientProvider } from './KubernetesClientProvider'; +import { ClusterDetails } from '../types/types'; +import * as https from 'https'; +import mockFs from 'mock-fs'; describe('KubernetesClientProvider', () => { beforeEach(() => { jest.resetAllMocks(); }); + afterEach(() => { + mockFs.restore(); + }); - it('can get core client by cluster details', async () => { + it('can get core client by cluster details', () => { const sut = new KubernetesClientProvider(); + const getKubeConfig = jest.spyOn(sut, 'getKubeConfig'); - const mockGetKubeConfig = jest.fn(sut.getKubeConfig.bind({})); - - sut.getKubeConfig = mockGetKubeConfig; - - const result = sut.getCoreClientByClusterDetails({ + const clusterDetails: ClusterDetails = { name: 'cluster-name', url: 'http://localhost:9999', serviceAccountToken: 'TOKEN', authProvider: 'serviceAccount', - skipTLSVerify: false, - }); + }; + const result = sut.getCoreClientByClusterDetails(clusterDetails); expect(result.basePath).toBe('http://localhost:9999'); // These fields aren't on the type but are there @@ -44,23 +47,21 @@ describe('KubernetesClientProvider', () => { expect(auth.clusters[0].name).toBe('cluster-name'); expect(auth.clusters[0].skipTLSVerify).toBe(false); - expect(mockGetKubeConfig.mock.calls.length).toBe(1); + expect(getKubeConfig).toHaveBeenCalledTimes(1); }); - it('can get custom objects client by cluster details', async () => { + it('can get custom objects client by cluster details', () => { const sut = new KubernetesClientProvider(); + const getKubeConfig = jest.spyOn(sut, 'getKubeConfig'); - const mockGetKubeConfig = jest.fn(sut.getKubeConfig.bind({})); - - sut.getKubeConfig = mockGetKubeConfig; - - const result = sut.getCustomObjectsClient({ + const clusterDetails: ClusterDetails = { name: 'cluster-name', url: 'http://localhost:9999', serviceAccountToken: 'TOKEN', authProvider: 'serviceAccount', skipTLSVerify: false, - }); + }; + const result = sut.getCustomObjectsClient(clusterDetails); expect(result.basePath).toBe('http://localhost:9999'); // These fields aren't on the type but are there @@ -68,6 +69,27 @@ describe('KubernetesClientProvider', () => { expect(auth.users[0].token).toBe('TOKEN'); expect(auth.clusters[0].name).toBe('cluster-name'); - expect(mockGetKubeConfig.mock.calls.length).toBe(1); + expect(getKubeConfig).toHaveBeenCalledTimes(1); + }); + + it('respects caFile', async () => { + mockFs({ + '/path/to/ca.crt': 'my-ca', + }); + const clusterDetails: ClusterDetails = { + name: 'cluster-name', + url: 'https://localhost:9999', + authProvider: 'serviceAccount', + serviceAccountToken: 'TOKEN', + caFile: '/path/to/ca.crt', + }; + const kubeConfig = new KubernetesClientProvider().getKubeConfig( + clusterDetails, + ); + + const options: https.RequestOptions = {}; + await kubeConfig.applytoHTTPSOptions(options); + + expect(options.ca?.toString()).toEqual('my-ca'); }); }); diff --git a/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts b/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts index 70764fed66..14c8e422fc 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesClientProvider.ts @@ -31,12 +31,13 @@ import { ClusterDetails } from '../types/types'; */ export class KubernetesClientProvider { // visible for testing - getKubeConfig(clusterDetails: ClusterDetails) { + getKubeConfig(clusterDetails: ClusterDetails): KubeConfig { const cluster: Cluster = { name: clusterDetails.name, server: clusterDetails.url, skipTLSVerify: clusterDetails.skipTLSVerify || false, caData: clusterDetails.caData, + caFile: clusterDetails.caFile, }; // TODO configure diff --git a/plugins/kubernetes-backend/src/types/types.ts b/plugins/kubernetes-backend/src/types/types.ts index ed43935202..d76bfc8149 100644 --- a/plugins/kubernetes-backend/src/types/types.ts +++ b/plugins/kubernetes-backend/src/types/types.ts @@ -167,6 +167,7 @@ export interface ClusterDetails { */ skipMetricsLookup?: boolean; caData?: string | undefined; + caFile?: string | undefined; /** * Specifies the link to the Kubernetes dashboard managing this cluster. * @remarks diff --git a/yarn.lock b/yarn.lock index 1a493fa380..f059985151 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6742,6 +6742,7 @@ __metadata: http-proxy-middleware: ^2.0.6 lodash: ^4.17.21 luxon: ^3.0.0 + mock-fs: ^5.2.0 morgan: ^1.10.0 msw: ^0.49.0 node-fetch: ^2.6.7 @@ -29069,7 +29070,7 @@ __metadata: languageName: node linkType: hard -"mock-fs@npm:^5.1.0, mock-fs@npm:^5.1.1": +"mock-fs@npm:^5.1.0, mock-fs@npm:^5.1.1, mock-fs@npm:^5.2.0": version: 5.2.0 resolution: "mock-fs@npm:5.2.0" checksum: c25835247bd26fa4e0189addd61f98973f61a72741e4d2a5694b143a2069b84978443a7ac0fdb1a71aead99273ec22ff4e9c968de11bbd076db020264c5b8312 From facbae6feff8b6715b75722e499f4fe876a8d6f7 Mon Sep 17 00:00:00 2001 From: iris Date: Thu, 1 Dec 2022 17:00:32 +0800 Subject: [PATCH 062/437] Add last_updated_at in final_entities table Signed-off-by: iris --- ...5_add_last_updated_at_in_final_entities.js | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js diff --git a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js new file mode 100644 index 0000000000..9afd32b774 --- /dev/null +++ b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js @@ -0,0 +1,31 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +exports.up = async function up(knex) { + await knex.schema.table('final_entities', table => { + table.timestamp('last_updated_at').nullable(); + }); +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function down(knex) { + await knex.schema.table('final_entities', table => { + table.dropColumn('last_updated_at'); + }); +}; From 253acc8b4395b2bdac6afdf651aba3b8c15bdc5f Mon Sep 17 00:00:00 2001 From: iris Date: Thu, 1 Dec 2022 17:08:36 +0800 Subject: [PATCH 063/437] update last_updated_at when update final_entities Signed-off-by: iris --- plugins/catalog-backend/src/database/tables.ts | 1 + plugins/catalog-backend/src/stitching/Stitcher.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index c23753b267..e3f3efd1bb 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -66,6 +66,7 @@ export type DbFinalEntitiesRow = { hash: string; stitch_ticket: string; final_entity?: string; + last_updated_at?: string | Date; }; export type DbSearchRow = { diff --git a/plugins/catalog-backend/src/stitching/Stitcher.ts b/plugins/catalog-backend/src/stitching/Stitcher.ts index 0da8e39694..64f5527262 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.ts @@ -207,6 +207,7 @@ export class Stitcher { .update({ final_entity: JSON.stringify(entity), hash, + last_updated_at: this.database.fn.now(), }) .where('entity_id', entityId) .where('stitch_ticket', ticket) From a483d5f5a4d82219e4e41e764127020de02ea9c9 Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 2 Dec 2022 12:05:18 +0800 Subject: [PATCH 064/437] add last_updated_at as annotation backstage.io/last_updated-at Signed-off-by: iris --- .../src/service/DefaultEntitiesCatalog.ts | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 14f22f862b..1a0fe98b40 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -210,7 +210,14 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { }; } - let entities: Entity[] = rows.map(e => JSON.parse(e.final_entity!)); + let entities: Entity[] = rows.map(e => { + const entityJson = JSON.parse(e.final_entity!); + if (e.last_updated_at) { + entityJson.metadata.annotations['backstage.io/last_updated-at'] = + e.last_updated_at; + } + return entityJson; + }); if (request?.fields) { entities = entities.map(e => request.fields!(e)); @@ -263,7 +270,12 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { query = parseFilter(request.filter, query, this.database); } for (const row of await query) { - lookup.set(row.entityRef, row.entity ? JSON.parse(row.entity) : null); + const entityJson = JSON.parse(row.entity); + if (row.entity.last_updated_at) { + entityJson.metadata.annotations['backstage.io/last_updated-at'] = + row.entity.last_updated_at; + } + lookup.set(row.entityRef, row.entity ? entityJson : null); } } @@ -373,13 +385,18 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { .where('refresh_state.entity_ref', '=', rootRef) .select({ entityJson: 'final_entities.final_entity', + last_updated_at: 'final_entities.last_updated_at', }); if (!rootRow) { throw new NotFoundError(`No such entity ${rootRef}`); } - - const rootEntity = JSON.parse(rootRow.entityJson) as Entity; + const entityJson = JSON.parse(rootRow.entityJson); + if (rootRow.last_updated_at) { + entityJson.metadata.annotations['backstage.io/last_updated-at'] = + rootRow.last_updated_at; + } + const rootEntity = entityJson as Entity; const seenEntityRefs = new Set(); const todo = new Array(); const items = new Array<{ entity: Entity; parentEntityRefs: string[] }>(); From 93870e4df1a1a7d9f5f09fed5c333bcf5f97735f Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 2 Dec 2022 13:00:31 +0800 Subject: [PATCH 065/437] Add changeset Signed-off-by: iris --- .changeset/lucky-singers-worry.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/lucky-singers-worry.md diff --git a/.changeset/lucky-singers-worry.md b/.changeset/lucky-singers-worry.md new file mode 100644 index 0000000000..742883e784 --- /dev/null +++ b/.changeset/lucky-singers-worry.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': major +--- + +Track the last time the final entity changed with new timestamp "last_updated_at" data in final_entities database, which gets updated with the time when final_entities is updated. And it's displayed in metadata annotation as backstage.io/last_updated-at. From 06f6a4f0f14de66fe5372e5f9d773e9076ee3e96 Mon Sep 17 00:00:00 2001 From: Scott Guymer Date: Fri, 2 Dec 2022 16:48:22 +0100 Subject: [PATCH 066/437] fix: allow overriding of stackoverflow configuration Make it possible to override the config when instantiating for more flexibility. Signed-off-by: Scott Guymer --- .changeset/olive-eyes-sing.md | 5 +++++ .../src/search/StackOverflowQuestionsCollatorFactory.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/olive-eyes-sing.md diff --git a/.changeset/olive-eyes-sing.md b/.changeset/olive-eyes-sing.md new file mode 100644 index 0000000000..d0ae1dc08f --- /dev/null +++ b/.changeset/olive-eyes-sing.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-stack-overflow-backend': minor +--- + +Enable configuration override for StackOverflow backend plugin when instantiating the search indexer. This makes it possible to set different configuration for frontend and backend of the plugin. diff --git a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts index ebbad99d0a..807d6637c1 100644 --- a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts +++ b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts @@ -95,11 +95,11 @@ export class StackOverflowQuestionsCollatorFactory 'https://api.stackexchange.com/2.2'; const maxPage = options.maxPage || 100; return new StackOverflowQuestionsCollatorFactory({ - ...options, baseUrl, maxPage, apiKey, apiAccessToken, + ...options, }); } From 6151f8e07166a25e7940a303609ab11bdd91adcc Mon Sep 17 00:00:00 2001 From: Scott Guymer Date: Mon, 5 Dec 2022 09:19:49 +0100 Subject: [PATCH 067/437] Added some basic tests for stackoverflow backend plugin Signed-off-by: Scott Guymer --- plugins/stack-overflow-backend/package.json | 8 +- ...ckOverflowQuestionsCollatorFactory.test.ts | 151 ++++++++++++++++++ 2 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts diff --git a/plugins/stack-overflow-backend/package.json b/plugins/stack-overflow-backend/package.json index 5a58f54958..1750ddbc6c 100644 --- a/plugins/stack-overflow-backend/package.json +++ b/plugins/stack-overflow-backend/package.json @@ -32,13 +32,19 @@ "clean": "backstage-cli package clean" }, "dependencies": { - "@backstage/cli": "workspace:^", + "@backstage/backend-common": "workspace:^", "@backstage/config": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "node-fetch": "^2.6.7", "qs": "^6.9.4", "winston": "^3.2.1" }, + "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", + "@backstage/cli": "workspace:^", + "@backstage/plugin-search-backend-node": "workspace:^", + "msw": "^0.49.0" + }, "files": [ "dist", "config.d.ts" diff --git a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts new file mode 100644 index 0000000000..9f2c316be4 --- /dev/null +++ b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { getVoidLogger } from '@backstage/backend-common'; +import { + StackOverflowQuestionsCollatorFactory, + StackOverflowQuestionsCollatorFactoryOptions, +} from './StackOverflowQuestionsCollatorFactory'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { ConfigReader } from '@backstage/config'; +import { Readable } from 'stream'; +import { setupServer } from 'msw/node'; +import { rest } from 'msw'; + +const logger = getVoidLogger(); + +const mockQuestion = { + items: [ + { + tags: ['backstage'], + owner: { + display_name: 'The Riddler', + }, + answer_count: 1, + link: 'https://stack.overflow.local/questions/2911', + title: 'This is the first question', + }, + ], + has_more: false, +}; + +const mockOverrideQuestion = { + items: [ + { + tags: ['backstage'], + owner: { + display_name: 'The Riddler', + }, + answer_count: 1, + link: 'https://stack.overflow.local/questions/1', + title: 'This is the first question', + }, + { + tags: ['backstage'], + owner: { + display_name: 'The Riddler', + }, + answer_count: 1, + link: 'https://stack.overflow.local/questions/2', + title: 'this is another question', + }, + ], + has_more: false, +}; + +describe('StackOverflowQuestionsCollatorFactory', () => { + const config = new ConfigReader({ + stackoverflow: { + baseUrl: 'http://stack.overflow.local', + }, + }); + + const defaultOptions: StackOverflowQuestionsCollatorFactoryOptions = { + logger, + requestParams: { + tagged: ['developer-portal'], + pagesize: 100, + order: 'desc', + sort: 'activity', + }, + }; + + it('has expected type', () => { + const factory = StackOverflowQuestionsCollatorFactory.fromConfig( + config, + defaultOptions, + ); + expect(factory.type).toBe('stack-overflow'); + }); + + describe('getCollator', () => { + const worker = setupServer(); + setupRequestMockHandlers(worker); + + afterEach(async () => { + worker.resetHandlers(); + }); + + afterAll(async () => { + worker.close(); + }); + + it('returns a readable stream', async () => { + const factory = StackOverflowQuestionsCollatorFactory.fromConfig( + config, + defaultOptions, + ); + const collator = await factory.getCollator(); + expect(collator).toBeInstanceOf(Readable); + }); + + it('fetches from the configured endpoint', async () => { + worker.use( + rest.get('http://stack.overflow.local/questions', (_, res, ctx) => + res(ctx.status(200), ctx.json(mockQuestion)), + ), + ); + const factory = StackOverflowQuestionsCollatorFactory.fromConfig( + config, + defaultOptions, + ); + const collator = await factory.getCollator(); + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + + expect(documents).toHaveLength(mockQuestion.items.length); + }); + + it('fetches from the overridden endpoint', async () => { + worker.use( + rest.get('http://stack.overflow.override/questions', (_, res, ctx) => + res(ctx.status(200), ctx.json(mockOverrideQuestion)), + ), + ); + const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { + logger, + baseUrl: 'http://stack.overflow.override', + requestParams: defaultOptions.requestParams, + }); + const collator = await factory.getCollator(); + + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + + expect(documents).toHaveLength(mockOverrideQuestion.items.length); + }); + }); +}); From 9e4725f8f4736dbccccef47a8ed640abc383e5c0 Mon Sep 17 00:00:00 2001 From: Scott Guymer Date: Mon, 5 Dec 2022 09:38:20 +0100 Subject: [PATCH 068/437] Updated yarn.lock Signed-off-by: Scott Guymer --- yarn.lock | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/yarn.lock b/yarn.lock index 4af9e1862a..9b8f80db7e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7845,9 +7845,13 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/plugin-stack-overflow-backend@workspace:plugins/stack-overflow-backend" dependencies: + "@backstage/backend-common": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" + msw: ^0.49.0 node-fetch: ^2.6.7 qs: ^6.9.4 winston: ^3.2.1 From 68cf4f593416e71248c4e38f9654bfccc351f83c Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 2 Dec 2022 13:02:43 +0800 Subject: [PATCH 069/437] fix testing Signed-off-by: iris --- .changeset/lucky-singers-worry.md | 2 +- ...5_add_last_updated_at_in_final_entities.js | 5 ++- .../catalog-backend/src/database/tables.ts | 2 +- .../src/service/DefaultEntitiesCatalog.ts | 6 ++-- .../src/stitching/Stitcher.test.ts | 35 +++++++++++++++++-- .../catalog-backend/src/stitching/Stitcher.ts | 3 +- 6 files changed, 44 insertions(+), 9 deletions(-) diff --git a/.changeset/lucky-singers-worry.md b/.changeset/lucky-singers-worry.md index 742883e784..abe581fbc3 100644 --- a/.changeset/lucky-singers-worry.md +++ b/.changeset/lucky-singers-worry.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend': major --- -Track the last time the final entity changed with new timestamp "last_updated_at" data in final_entities database, which gets updated with the time when final_entities is updated. And it's displayed in metadata annotation as backstage.io/last_updated-at. +Track the last time the final entity changed with new timestamp "last updated at" data in final entities database, which gets updated with the time when final entity is updated. And it's displayed in metadata annotation. diff --git a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js index 9afd32b774..d12d5bec3e 100644 --- a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js +++ b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js @@ -16,7 +16,10 @@ exports.up = async function up(knex) { await knex.schema.table('final_entities', table => { - table.timestamp('last_updated_at').nullable(); + table + .bigint('last_updated_at') + .nullable() + .comment('The time when final_entity changed'); }); }; diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index e3f3efd1bb..2778cf2fba 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -66,7 +66,7 @@ export type DbFinalEntitiesRow = { hash: string; stitch_ticket: string; final_entity?: string; - last_updated_at?: string | Date; + last_updated_at: string | null; }; export type DbSearchRow = { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 1a0fe98b40..a305df52e3 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -213,7 +213,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { let entities: Entity[] = rows.map(e => { const entityJson = JSON.parse(e.final_entity!); if (e.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated-at'] = + entityJson.metadata.annotations['backstage.io/last_updated_at'] = e.last_updated_at; } return entityJson; @@ -272,7 +272,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { for (const row of await query) { const entityJson = JSON.parse(row.entity); if (row.entity.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated-at'] = + entityJson.metadata.annotations['backstage.io/last_updated_at'] = row.entity.last_updated_at; } lookup.set(row.entityRef, row.entity ? entityJson : null); @@ -393,7 +393,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } const entityJson = JSON.parse(rootRow.entityJson); if (rootRow.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated-at'] = + entityJson.metadata.annotations['backstage.io/last_updated_at'] = rootRow.last_updated_at; } const rootEntity = entityJson as Entity; diff --git a/plugins/catalog-backend/src/stitching/Stitcher.test.ts b/plugins/catalog-backend/src/stitching/Stitcher.test.ts index a3657e45fd..a13467e9bd 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.test.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.test.ts @@ -17,6 +17,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { TestDatabases } from '@backstage/backend-test-utils'; import { Entity } from '@backstage/catalog-model'; +import { DateTime } from 'luxon'; import { applyDatabaseMigrations } from '../database/migrations'; import { DbFinalEntitiesRow, @@ -42,6 +43,7 @@ describe('Stitcher', () => { const stitcher = new Stitcher(db, logger); let entities: DbFinalEntitiesRow[]; let entity: Entity; + const timeBeforeStitch = DateTime.now(); await db('refresh_state').insert([ { @@ -110,6 +112,15 @@ describe('Stitcher', () => { }); expect(entity.metadata.etag).toEqual(entities[0].hash); + const last_updated_at = entities[0].last_updated_at; + expect(last_updated_at).not.toBeNull(); + const lastUpdatedAt = DateTime.fromMillis( + last_updated_at ? +last_updated_at : 0, + ); + const msAfterStitch = lastUpdatedAt + .diff(timeBeforeStitch, 'milliseconds') + .toObject(); + expect(msAfterStitch.milliseconds).toBeGreaterThan(0); const firstHash = entities[0].hash; const search = await db('search'); @@ -127,7 +138,12 @@ describe('Stitcher', () => { original_value: 'a', value: 'a', }, - { entity_id: 'my-id', key: 'kind', original_value: 'k', value: 'k' }, + { + entity_id: 'my-id', + key: 'kind', + original_value: 'k', + value: 'k', + }, { entity_id: 'my-id', key: 'metadata.name', @@ -174,6 +190,7 @@ describe('Stitcher', () => { }, ]); + const timeBeforeRestitch = DateTime.now(); await stitcher.stitch(new Set(['k:ns/n'])); entities = await db('final_entities'); @@ -206,6 +223,15 @@ describe('Stitcher', () => { expect(entities[0].hash).not.toEqual(firstHash); expect(entities[0].hash).toEqual(entity.metadata.etag); + expect(entity.metadata.etag).toEqual(entities[0].hash); + const last_updated_at_after_restitch = entities[0].last_updated_at; + expect(last_updated_at_after_restitch).not.toBeNull(); + const msAfterRestitch = DateTime.fromMillis( + last_updated_at_after_restitch ? +last_updated_at_after_restitch : 0, + ) + .diff(timeBeforeRestitch, 'milliseconds') + .toObject(); + expect(msAfterRestitch.milliseconds).toBeGreaterThan(0); expect(await db('search')).toEqual( expect.arrayContaining([ @@ -227,7 +253,12 @@ describe('Stitcher', () => { original_value: 'a', value: 'a', }, - { entity_id: 'my-id', key: 'kind', original_value: 'k', value: 'k' }, + { + entity_id: 'my-id', + key: 'kind', + original_value: 'k', + value: 'k', + }, { entity_id: 'my-id', key: 'metadata.name', diff --git a/plugins/catalog-backend/src/stitching/Stitcher.ts b/plugins/catalog-backend/src/stitching/Stitcher.ts index 64f5527262..c3892244dd 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.ts @@ -23,6 +23,7 @@ import { import { SerializedError, stringifyError } from '@backstage/errors'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; +import { DateTime } from 'luxon'; import { Logger } from 'winston'; import { DbFinalEntitiesRow, @@ -207,7 +208,7 @@ export class Stitcher { .update({ final_entity: JSON.stringify(entity), hash, - last_updated_at: this.database.fn.now(), + last_updated_at: `${DateTime.now().toMillis()}`, }) .where('entity_id', entityId) .where('stitch_ticket', ticket) From b35c1770cf61beb8177314db8510092282da7852 Mon Sep 17 00:00:00 2001 From: Juan Pablo Garcia Ripa Date: Mon, 5 Dec 2022 19:25:40 +0100 Subject: [PATCH 070/437] change path argument to options and allow csv and glob for paths Signed-off-by: Juan Pablo Garcia Ripa --- .changeset/lemon-coats-camp.md | 7 +- package.json | 2 +- packages/repo-tools/cli-report.md | 9 +- packages/repo-tools/package.json | 2 + .../src/commands/api-reports/api-extractor.ts | 40 ++---- .../src/commands/api-reports/api-reports.ts | 122 ++++++++++++------ packages/repo-tools/src/commands/index.ts | 14 +- packages/repo-tools/src/lib/paths.test.ts | 51 +++++++- packages/repo-tools/src/lib/paths.ts | 25 ++++ yarn.lock | 2 + 10 files changed, 187 insertions(+), 87 deletions(-) diff --git a/.changeset/lemon-coats-camp.md b/.changeset/lemon-coats-camp.md index a2d87476f6..ae31098e2f 100644 --- a/.changeset/lemon-coats-camp.md +++ b/.changeset/lemon-coats-camp.md @@ -4,8 +4,9 @@ Add new command options to the `api-report` -- added `--allowWarnings` to continue processing packages if some packages have warnings -- added `--omitMessages` to pass some warnings messages code to be omitted from the api-report.md files +- added `--allow-warnings`, `-a` to continue processing packages if some packages have warnings +- added `--omit-messages`, `-o` to pass some warnings messages code to be omitted from the api-report.md files +- added `--paths`, `-p` to select packages path to process - The `paths` argument for this command now takes as default the value on `workspaces.packages` inside the root package.json -- The `paths` argument now allow glob patterns +- Removed the `paths` argument replaced by the option `--paths` - change the path resolution to use the `@backstage/cli-common` packages instead diff --git a/package.json b/package.json index 497a652cdf..5182436762 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "build:backend": "yarn workspace backend build", "build:all": "backstage-cli repo build --all", "build:api-reports": "yarn build:api-reports:only --tsc", - "build:api-reports:only": "backstage-repo-tools api-reports --allow-warnings packages/core-components plugins/catalog plugins/catalog-import plugins/git-release-manager plugins/jenkins plugins/kubernetes", + "build:api-reports:only": "backstage-repo-tools api-reports --allow-warnings 'packages/core-components,plugins/+(catalog|catalog-import|git-release-manager|jenkins|kubernetes)'", "build:api-docs": "LANG=en_EN yarn build:api-reports --docs", "tsc": "tsc", "tsc:full": "backstage-cli repo clean && tsc --skipLibCheck false --incremental false", diff --git a/packages/repo-tools/cli-report.md b/packages/repo-tools/cli-report.md index ba15ecc674..d3446e0262 100644 --- a/packages/repo-tools/cli-report.md +++ b/packages/repo-tools/cli-report.md @@ -12,7 +12,7 @@ Options: -h, --help Commands: - api-reports [options] [paths...] + api-reports [options] type-deps help [command] ``` @@ -20,14 +20,15 @@ Commands: ### `backstage-repo-tools api-reports` ``` -Usage: backstage-repo-tools api-reports [options] [paths...] +Usage: backstage-repo-tools api-reports [options] Options: + -p --paths [paths...] --ci --tsc --docs - --allow-warnings [allowWarningsPaths...] - --omitMessages + -a, --allow-warnings [allowWarningsPaths...] + -o, --omit-messages -h, --help ``` diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index 9b51d35922..6025af9bce 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -40,7 +40,9 @@ "chalk": "^4.0.0", "commander": "^9.1.0", "fs-extra": "10.1.0", + "glob": "^8.0.3", "is-glob": "^4.0.3", + "minimatch": "^5.1.1", "ts-node": "^10.0.0" }, "devDependencies": { diff --git a/packages/repo-tools/src/commands/api-reports/api-extractor.ts b/packages/repo-tools/src/commands/api-reports/api-extractor.ts index b2fab40ca5..46f4ea984b 100644 --- a/packages/repo-tools/src/commands/api-reports/api-extractor.ts +++ b/packages/repo-tools/src/commands/api-reports/api-extractor.ts @@ -65,14 +65,9 @@ import { } from '@microsoft/api-documenter/lib/markdown/CustomMarkdownEmitter'; import { IMarkdownEmitterContext } from '@microsoft/api-documenter/lib/markdown/MarkdownEmitter'; import { AstDeclaration } from '@microsoft/api-extractor/lib/analyzer/AstDeclaration'; -import { paths as cliPaths, resolvePackagePath } from '../../lib/paths'; +import { paths as cliPaths } from '../../lib/paths'; -import g from 'glob'; -import isGlob from 'is-glob'; - -import { promisify } from 'util'; - -const glob = promisify(g); +import minimatch from 'minimatch'; const tmpDir = cliPaths.resolveTargetRoot( './node_modules/.cache/api-extractor', @@ -226,24 +221,6 @@ ApiReportGenerator.generateReviewFileContent = }); }; -export async function findPackageDirs(selectedPaths: string[]) { - const packageDirs = new Array(); - for (const packageRoot of selectedPaths) { - const fullPath = cliPaths.resolveTargetRoot(packageRoot); - - // if the path contain any glob notation we resolve all the paths to process one by one - const dirs = isGlob(fullPath) ? await glob(fullPath) : [fullPath]; - for (const dir of dirs) { - const packageDir = await resolvePackagePath(dir); - if (!packageDir) { - continue; - } - packageDirs.push(packageDir); - } - } - return packageDirs; -} - export async function createTemporaryTsConfig(includedPackageDirs: string[]) { const path = cliPaths.resolveTargetRoot('tsconfig.tmp.json'); @@ -331,7 +308,7 @@ interface ApiExtractionOptions { outputDir: string; isLocalBuild: boolean; tsconfigFilePath: string; - allowWarnings: boolean | string[]; + allowWarnings?: boolean | string[]; omitMessages?: string[]; } @@ -340,7 +317,7 @@ export async function runApiExtraction({ outputDir, isLocalBuild, tsconfigFilePath, - allowWarnings, + allowWarnings = false, omitMessages = [], }: ApiExtractionOptions) { await fs.remove(outputDir); @@ -366,7 +343,7 @@ export async function runApiExtraction({ for (const packageDir of packageDirs) { console.log(`## Processing ${packageDir}`); const noBail = Array.isArray(allowWarnings) - ? allowWarnings.includes(packageDir) + ? allowWarnings.some(aw => aw === packageDir || minimatch(packageDir, aw)) : allowWarnings; const projectFolder = cliPaths.resolveTargetRoot(packageDir); @@ -514,6 +491,9 @@ export async function runApiExtraction({ } const warningCountAfter = await countApiReportWarnings(projectFolder); + if (noBail) { + console.log(`Skipping warnings check for ${packageDir}`); + } if (warningCountAfter > 0 && !noBail) { throw new Error( `The API Report for ${packageDir} is not allowed to have warnings`, @@ -1289,13 +1269,11 @@ function generateCliReport(name: string, models: CliModel[]): string { } interface CliExtractionOptions { - projectRoot: string; packageDirs: string[]; isLocalBuild: boolean; } export async function runCliExtraction({ - projectRoot, packageDirs, isLocalBuild, }: CliExtractionOptions) { @@ -1344,7 +1322,7 @@ export async function runCliExtraction({ console.log(''); console.log( `The conflicting file is ${relativePath( - projectRoot, + cliPaths.targetRoot, reportPath, )}, expecting the following content:`, ); diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index 9b0d78d6a8..11b65823c7 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -15,49 +15,42 @@ */ import { OptionValues } from 'commander'; -import { resolve as resolvePath } from 'path'; import fs from 'fs-extra'; import { spawnSync } from 'child_process'; import { createTemporaryTsConfig, - findPackageDirs, categorizePackageDirs, runApiExtraction, runCliExtraction, buildDocs, } from './api-extractor'; -import { paths as cliPaths } from '../../lib/paths'; +import { findPackageDirs, paths as cliPaths } from '../../lib/paths'; -export default async (paths: string[], opts: OptionValues) => { - const tmpDir = resolvePath( - cliPaths.targetRoot, +export default async (opts: OptionValues) => { + const tmpDir = cliPaths.resolveTargetRoot( './node_modules/.cache/api-extractor', ); - const projectRoot = resolvePath(cliPaths.targetRoot); const isCiBuild = opts.ci; const isDocsBuild = opts.docs; const runTsc = opts.tsc; - const selectedPaths = paths.length ? paths : await getWorkspacePkgs(); - const allowWarnings: boolean | string[] = opts.allowWarnings; - const omitMessages = opts.omitMessages; + const parsedPaths = parseArrayOption(opts.paths); + const isAllPackages = !Array.isArray(parsedPaths) || !parsedPaths?.length; + const selectedPaths = isAllPackages ? await getWorkspacePkgs() : parsedPaths; const selectedPackageDirs = await findPackageDirs(selectedPaths); - if (paths.length && isCiBuild) { - // TODO @sarabadu we can remove this validation to allow `/plugins/*` on CI?? - throw new Error( - 'Package path arguments are not supported together with the --ci flag', - ); - } - if (!paths.length && !isCiBuild && !isDocsBuild) { + const allowWarnings = parseArrayOption(opts.allowWarnings); + const omitMessages = parseArrayOption(opts.omitMessages); + + if (isAllPackages && !isCiBuild && !isDocsBuild) { console.log(''); console.log( 'TIP: You can generate api-reports for select packages by passing package paths:', ); console.log(''); console.log( - ' yarn build:api-reports packages/config packages/core-plugin-api plugins/*', + ' yarn build:api-reports -p packages/config -p packages/core-plugin-api,plugins/*', ); console.log(''); } @@ -67,27 +60,11 @@ export default async (paths: string[], opts: OptionValues) => { temporaryTsConfigPath = await createTemporaryTsConfig(selectedPackageDirs); } const tsconfigFilePath = - temporaryTsConfigPath ?? resolvePath(projectRoot, 'tsconfig.json'); + temporaryTsConfigPath ?? cliPaths.resolveTargetRoot('tsconfig.json'); if (runTsc) { - await fs.remove(resolvePath(projectRoot, 'dist-types')); - const { status } = spawnSync( - 'yarn', - [ - 'tsc', - ['--project', tsconfigFilePath], - ['--skipLibCheck', 'false'], - ['--incremental', 'false'], - ].flat(), - { - stdio: 'inherit', - shell: true, - cwd: projectRoot, - }, - ); - if (status !== 0) { - process.exit(status || undefined); - } + console.log('# Compiling TypeScript'); + await generateTSC(tsconfigFilePath); } const { tsPackageDirs, cliPackageDirs } = await categorizePackageDirs( @@ -102,13 +79,12 @@ export default async (paths: string[], opts: OptionValues) => { isLocalBuild: !isCiBuild, tsconfigFilePath, allowWarnings, - omitMessages, + omitMessages: Array.isArray(omitMessages) ? omitMessages : [], }); } if (cliPackageDirs.length > 0) { console.log('# Generating package CLI reports'); await runCliExtraction({ - projectRoot, packageDirs: cliPackageDirs, isLocalBuild: !isCiBuild, }); @@ -118,10 +94,49 @@ export default async (paths: string[], opts: OptionValues) => { console.log('# Generating package documentation'); await buildDocs({ inputDir: tmpDir, - outputDir: resolvePath(projectRoot, 'docs/reference'), + outputDir: cliPaths.resolveTargetRoot('docs/reference'), }); } }; + +/** + * Generates the TypeScript declaration files for the specified project, using the provided `tsconfig.json` file. + * + * Any existing declaration files in the `dist-types` directory will be deleted before generating the new ones. + * + * If the `tsc` command exits with a non-zero exit code, the process will be terminated with the same exit code. + * + * @param tsconfigFilePath {string} The path to the `tsconfig.json` file to use for generating the declaration files. + * @returns {Promise} A promise that resolves when the declaration files have been generated. + */ +export async function generateTSC(tsconfigFilePath: string) { + await fs.remove(cliPaths.resolveTargetRoot('dist-types')); + const { status } = spawnSync( + 'yarn', + [ + 'tsc', + ['--project', tsconfigFilePath], + ['--skipLibCheck', 'false'], + ['--incremental', 'false'], + ].flat(), + { + stdio: 'inherit', + shell: true, + cwd: cliPaths.targetRoot, + }, + ); + if (status !== 0) { + process.exit(status || undefined); + } +} + +/** + * Retrieves the list of package names in the "workspaces" field of the `package.json` file in the current workspace root. + * + * If the file does not exist, or the "workspaces" field is not present, returns `undefined`. + * + * @returns {Promise} The list of package names, or `undefined` if not found. + */ async function getWorkspacePkgs() { const pkgJson = await fs .readJson(cliPaths.resolveTargetRoot('package.json')) @@ -134,3 +149,30 @@ async function getWorkspacePkgs() { const workspaces = pkgJson?.workspaces?.packages; return workspaces; } + +/** + * Splits each string in the input array on comma, and returns an array of the resulting substrings. + * If the input array is `undefined`, returns `undefined`. If the input value is `true` or `false`, + * returns the value as-is. + * + * @param value An array of strings to be split on comma, or a boolean value (inherithed from commanderjs array args). + * @returns An array of the resulting substrings, the original boolean value, or `undefined` if the input value is `undefined`. + * + * @example + * parseOption(['foo,bar,baz']) + * // returns ['foo', 'bar', 'baz'] + * + * parseOption(true) + * // returns true + * + * parseOption() + * // returns undefined + */ +function parseArrayOption(value: string[] | boolean | undefined) { + if (typeof value === 'boolean') { + return value; + } + return value?.flatMap((str: string) => + str.includes(',') ? str.split(',') : str, + ); +} diff --git a/packages/repo-tools/src/commands/index.ts b/packages/repo-tools/src/commands/index.ts index bbc6773f83..5742ce7241 100644 --- a/packages/repo-tools/src/commands/index.ts +++ b/packages/repo-tools/src/commands/index.ts @@ -21,21 +21,21 @@ import { exitWithError } from '../lib/errors'; export function registerCommands(program: Command) { program .command('api-reports') - .argument( - '[paths...]', - 'path of package folder to extract API reports, `workspaces.packages` from root packages.json by default', + .option( + '-p --paths [paths...]', + 'paths of package folder to extract API reports, `workspaces.packages` from root packages.json by default. Allows glob patterns and comma separated values', ) .option('--ci', 'CI run checks that there is no changes on API reports') .option('--tsc', 'executes the tsc compilation before extracting the APIs') .option('--docs', 'generates the api documentation') .option( - '--allow-warnings [allowWarningsPaths...]', - 'continue processing packages after getting errors on selected packages', + '-a, --allow-warnings [allowWarningsPaths...]', + 'continue processing packages after getting errors on selected packages Allows glob patterns and comma separated values (i.e. packages/core,plugins/core-*)', false, ) .option( - '--omitMessages ', - 'select some message code to be omited on the API Extractor (i.e ae-cyclic-inherit-doc)', + '-o, --omit-messages ', + 'select some message code to be omited on the API Extractor (comma separated values i.e ae-cyclic-inherit-doc,ae-missing-getter )', ) .description('Generate an API report for selected packages') .action( diff --git a/packages/repo-tools/src/lib/paths.test.ts b/packages/repo-tools/src/lib/paths.test.ts index 90d84aec3c..c642900612 100644 --- a/packages/repo-tools/src/lib/paths.test.ts +++ b/packages/repo-tools/src/lib/paths.test.ts @@ -16,7 +16,7 @@ import mockFs from 'mock-fs'; import { resolve as resolvePath } from 'path'; -import { resolvePackagePath, paths } from './paths'; +import { resolvePackagePath, paths, findPackageDirs } from './paths'; describe('paths', () => { jest.spyOn(paths, 'targetRoot', 'get').mockReturnValue('/root'); @@ -38,6 +38,14 @@ describe('paths', () => { 'package-c': {}, 'README.md': 'Hello World', }, + plugins: { + 'plugin-a': { + 'package.json': '{}', + }, + 'plugin-b': { + 'package.json': '{}', + }, + }, }, }); }); @@ -59,8 +67,49 @@ describe('paths', () => { 'packages/package-b', ); }); + it('should work with absolute paths', async () => { + expect(await resolvePackagePath('/root/packages/package-a')).toBe( + 'packages/package-a', + ); + }); it('should return undefined if the pat is not a directory', async () => { expect(await resolvePackagePath('packages/README.md')).toBeUndefined(); }); }); + describe('findPackageDirs', () => { + it('should return only the given packages', async () => { + expect(await findPackageDirs(['packages/package-a'])).toEqual([ + 'packages/package-a', + ]); + }); + it('should return only the given packages when using glob patterns', async () => { + expect(await findPackageDirs(['packages/*'])).toEqual([ + 'packages/package-a', + 'packages/package-b', + ]); + expect(await findPackageDirs(['packages/*', 'plugins/*'])).toEqual([ + 'packages/package-a', + 'packages/package-b', + 'plugins/plugin-a', + 'plugins/plugin-b', + ]); + }); + it('should return only the given packages when using absolute paths', async () => { + expect( + await findPackageDirs([ + '/root/packages/package-a', + '/root/plugins/plugin-b', + ]), + ).toEqual(['packages/package-a', 'plugins/plugin-b']); + }); + it('should return only the given packages when using absolute paths with glob patterns', async () => { + expect( + await findPackageDirs(['/root/packages/*', '/root/plugins/*-a']), + ).toEqual([ + 'packages/package-a', + 'packages/package-b', + 'plugins/plugin-a', + ]); + }); + }); }); diff --git a/packages/repo-tools/src/lib/paths.ts b/packages/repo-tools/src/lib/paths.ts index f387e9be51..874bb48ea1 100644 --- a/packages/repo-tools/src/lib/paths.ts +++ b/packages/repo-tools/src/lib/paths.ts @@ -18,6 +18,13 @@ import { findPaths } from '@backstage/cli-common'; import { relative as relativePath, join } from 'path'; import fs from 'fs-extra'; +import g from 'glob'; +import isGlob from 'is-glob'; + +import { promisify } from 'util'; + +const glob = promisify(g); + /* eslint-disable-next-line no-restricted-syntax */ export const paths = findPaths(__dirname); @@ -41,3 +48,21 @@ export async function resolvePackagePath( } return relativePath(paths.targetRoot, fullPackageDir); } + +export async function findPackageDirs(selectedPaths: string[] = []) { + const packageDirs = new Array(); + for (const packageRoot of selectedPaths) { + const fullPath = paths.resolveTargetRoot(packageRoot); + + // if the path contain any glob notation we resolve all the paths to process one by one + const dirs = isGlob(fullPath) ? await glob(fullPath) : [fullPath]; + for (const dir of dirs) { + const packageDir = await resolvePackagePath(dir); + if (!packageDir) { + continue; + } + packageDirs.push(packageDir); + } + } + return packageDirs; +} diff --git a/yarn.lock b/yarn.lock index e92df381ca..cc9dcb0642 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8480,7 +8480,9 @@ __metadata: chalk: ^4.0.0 commander: ^9.1.0 fs-extra: 10.1.0 + glob: ^8.0.3 is-glob: ^4.0.3 + minimatch: ^5.1.1 mock-fs: ^5.1.0 ts-node: ^10.0.0 bin: From e5dc85a82139b42811f59bfe31058ead200c0285 Mon Sep 17 00:00:00 2001 From: Juan Pablo Garcia Ripa Date: Tue, 6 Dec 2022 11:12:07 +0100 Subject: [PATCH 071/437] adding tests to each option Signed-off-by: Juan Pablo Garcia Ripa --- .../commands/api-reports/api-reports.test.ts | 505 ++++++++++++++++++ .../src/commands/api-reports/api-reports.ts | 47 +- .../src/commands/api-reports/generateTSC.ts | 50 ++ packages/repo-tools/src/commands/index.ts | 4 +- 4 files changed, 571 insertions(+), 35 deletions(-) create mode 100644 packages/repo-tools/src/commands/api-reports/api-reports.test.ts create mode 100644 packages/repo-tools/src/commands/api-reports/generateTSC.ts diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.test.ts b/packages/repo-tools/src/commands/api-reports/api-reports.test.ts new file mode 100644 index 0000000000..fa5bfd6973 --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/api-reports.test.ts @@ -0,0 +1,505 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import mockFs from 'mock-fs'; +import { resolve as resolvePath } from 'path'; +import * as pathsLib from '../../lib/paths'; + +import { + buildDocs, + runCliExtraction, + runApiExtraction, + categorizePackageDirs, +} from './api-extractor'; + +import { buildApiReports } from './api-reports'; +import { generateTSC } from './generateTSC'; + +jest.mock('./generateTSC'); +// create mocks for the dependencies of the `buildApiReports` function +jest.mock('./api-extractor', () => ({ + createTemporaryTsConfig: jest.fn(), + categorizePackageDirs: jest.fn().mockImplementation(async (p: string[]) => { + console.log('categorizePackageDirs', p); + return { + tsPackageDirs: p, + cliPackageDirs: p, + }; + }), + runApiExtraction: jest.fn(), + runCliExtraction: jest.fn(), + buildDocs: jest.fn(), +})); + +const paths = pathsLib.paths; + +jest.spyOn(paths, 'targetRoot', 'get').mockReturnValue('/root'); +jest.spyOn(paths, 'resolveTargetRoot').mockImplementation((...path) => { + return resolvePath('/root', ...path); +}); + +describe('buildApiReports', () => { + beforeEach(() => { + mockFs({ + [paths.targetRoot]: { + 'package.json': JSON.stringify({ + workspaces: { packages: ['packages/*', 'plugins/*'] }, + }), + packages: { + 'package-a': { + 'package.json': '{}', + }, + 'package-b': { + 'package.json': '{}', + }, + 'package-c': {}, + 'README.md': 'Hello World', + }, + plugins: { + 'plugin-a': { + 'package.json': '{}', + }, + 'plugin-b': { + 'package.json': '{}', + }, + 'plugin-c': { + 'package.json': '{}', + }, + }, + }, + }); + }); + + afterEach(() => { + mockFs.restore(); + jest.clearAllMocks(); + }); + it('should run whitout any options', async () => { + const opts = {}; + + await buildApiReports(opts); + + expect(categorizePackageDirs).toHaveBeenCalledWith([ + 'packages/package-a', + 'packages/package-b', + 'plugins/plugin-a', + 'plugins/plugin-b', + 'plugins/plugin-c', + ]); + + expect(generateTSC).not.toHaveBeenCalled(); + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: [ + 'packages/package-a', + 'packages/package-b', + 'plugins/plugin-a', + 'plugins/plugin-b', + 'plugins/plugin-c', + ], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: undefined, + omitMessages: [], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + expect(runCliExtraction).toHaveBeenCalledWith({ + packageDirs: [ + 'packages/package-a', + 'packages/package-b', + 'plugins/plugin-a', + 'plugins/plugin-b', + + 'plugins/plugin-c', + ], + isLocalBuild: true, + }); + + expect(buildDocs).not.toHaveBeenCalled(); + }); + + describe('paths', () => { + it('should generate API reports for one specific package', async () => { + const opts = { + paths: ['packages/package-a'], + }; + + await buildApiReports(opts); + + expect(categorizePackageDirs).toHaveBeenCalledWith([ + 'packages/package-a', + ]); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a'], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: undefined, + omitMessages: [], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + expect(runCliExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a'], + isLocalBuild: true, + }); + + expect(buildDocs).not.toHaveBeenCalled(); + }); + it('should generate API reports for multiple specific packages', async () => { + const opts = { + paths: ['packages/package-a', 'packages/package-b'], + }; + + await buildApiReports(opts); + + expect(categorizePackageDirs).toHaveBeenCalledWith([ + 'packages/package-a', + 'packages/package-b', + ]); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: undefined, + omitMessages: [], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + expect(runCliExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + isLocalBuild: true, + }); + + expect(buildDocs).not.toHaveBeenCalled(); + }); + it('should generate API reports for all packages matching the glob pattern', async () => { + const opts = { + paths: ['packages/*'], + }; + + await buildApiReports(opts); + + expect(categorizePackageDirs).toHaveBeenCalledWith([ + 'packages/package-a', + 'packages/package-b', + ]); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: undefined, + omitMessages: [], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + expect(runCliExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + isLocalBuild: true, + }); + + expect(buildDocs).not.toHaveBeenCalled(); + }); + + it('should generate API reports for all packages matching multiple glob patterns', async () => { + const opts = { + paths: ['packages/*', 'plugins/*a'], + }; + + await buildApiReports(opts); + + expect(categorizePackageDirs).toHaveBeenCalledWith([ + 'packages/package-a', + 'packages/package-b', + 'plugins/plugin-a', + ]); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: [ + 'packages/package-a', + 'packages/package-b', + 'plugins/plugin-a', + ], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: undefined, + omitMessages: [], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + expect(runCliExtraction).toHaveBeenCalledWith({ + packageDirs: [ + 'packages/package-a', + 'packages/package-b', + 'plugins/plugin-a', + ], + isLocalBuild: true, + }); + + expect(buildDocs).not.toHaveBeenCalled(); + }); + + it('should generate API reports for specific packages and glob pattern', async () => { + const opts = { + paths: ['packages/package-a', 'plugins/*'], + }; + + await buildApiReports(opts); + + expect(categorizePackageDirs).toHaveBeenCalledWith([ + 'packages/package-a', + 'plugins/plugin-a', + 'plugins/plugin-b', + 'plugins/plugin-c', + ]); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: [ + 'packages/package-a', + 'plugins/plugin-a', + 'plugins/plugin-b', + 'plugins/plugin-c', + ], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: undefined, + omitMessages: [], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + expect(runCliExtraction).toHaveBeenCalledWith({ + packageDirs: [ + 'packages/package-a', + 'plugins/plugin-a', + 'plugins/plugin-b', + 'plugins/plugin-c', + ], + isLocalBuild: true, + }); + + expect(buildDocs).not.toHaveBeenCalled(); + }); + }); + describe('allowWarnings', () => { + it('should accept boolean values', async () => { + const opts = { + paths: ['packages/*'], + allowWarnings: true, + }; + + await buildApiReports(opts); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: true, + omitMessages: [], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + }); + + it('should accept single path value', async () => { + const opts = { + paths: ['packages/*'], + allowWarnings: ['packages/package-a'], + }; + + await buildApiReports(opts); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: ['packages/package-a'], + omitMessages: [], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + }); + + it('should accept multiple path values as array', async () => { + const opts = { + paths: ['packages/*'], + allowWarnings: ['packages/package-a', 'packages/package-b'], + }; + + await buildApiReports(opts); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: ['packages/package-a', 'packages/package-b'], + omitMessages: [], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + }); + + it('should accept multiple path values as comma separated string', async () => { + const opts = { + paths: ['packages/*'], + allowWarnings: ['packages/package-a,packages/package-b'], + }; + + await buildApiReports(opts); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: ['packages/package-a', 'packages/package-b'], + omitMessages: [], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + }); + + it('should accept multiple path values as comma separated string with spaces', async () => { + const opts = { + paths: ['packages/*'], + allowWarnings: ['packages/package-a, packages/package-b'], + }; + + await buildApiReports(opts); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: ['packages/package-a', 'packages/package-b'], + omitMessages: [], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + }); + }); + describe('omitMessages', () => { + it('should accept single message value', async () => { + const opts = { + paths: ['packages/*'], + omitMessages: ['ae-missing-release-tag'], + }; + + await buildApiReports(opts); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: undefined, + omitMessages: ['ae-missing-release-tag'], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + }); + + it('should accept multiple message values as array', async () => { + const opts = { + paths: ['packages/*'], + omitMessages: ['ae-missing-release-tag', 'ae-missing-annotations'], + }; + + await buildApiReports(opts); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: undefined, + omitMessages: ['ae-missing-release-tag', 'ae-missing-annotations'], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + }); + it('should accept multiple message values as comma separated string', async () => { + const opts = { + paths: ['packages/*'], + omitMessages: ['ae-missing-release-tag,ae-missing-annotations'], + }; + + await buildApiReports(opts); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: undefined, + omitMessages: ['ae-missing-release-tag', 'ae-missing-annotations'], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + }); + + it('should accept multiple message values as comma separated string with spaces', async () => { + const opts = { + paths: ['packages/*'], + omitMessages: ['ae-missing-release-tag, ae-missing-annotations'], + }; + + await buildApiReports(opts); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: undefined, + omitMessages: ['ae-missing-release-tag', 'ae-missing-annotations'], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + }); + }); + describe('isCI', () => { + it('should set localBuild to false if CI option is passed', async () => { + const opts = { + paths: ['packages/*'], + ci: true, + }; + + await buildApiReports(opts); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: undefined, + omitMessages: [], + isLocalBuild: false, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + expect(runCliExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + isLocalBuild: false, + }); + }); + }); + describe('docs', () => { + it('should run typedoc if docs option is passed', async () => { + const opts = { + paths: ['packages/*'], + docs: true, + }; + + await buildApiReports(opts); + + expect(buildDocs).toHaveBeenCalledWith({ + inputDir: '/root/node_modules/.cache/api-extractor', + outputDir: '/root/docs/reference', + }); + }); + }); + describe('tsc', () => { + it('should run tsc if tsc option is passed', async () => { + const opts = { + paths: ['packages/*'], + tsc: true, + }; + + await buildApiReports(opts); + + expect(generateTSC).toHaveBeenCalled(); + }); + }); +}); diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index 11b65823c7..b59f498d7b 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -16,7 +16,6 @@ import { OptionValues } from 'commander'; import fs from 'fs-extra'; -import { spawnSync } from 'child_process'; import { createTemporaryTsConfig, categorizePackageDirs, @@ -25,8 +24,18 @@ import { buildDocs, } from './api-extractor'; import { findPackageDirs, paths as cliPaths } from '../../lib/paths'; +import { generateTSC } from './generateTSC'; -export default async (opts: OptionValues) => { +type Options = { + ci?: boolean; + docs?: boolean; + tsc?: boolean; + paths?: string[]; + allowWarnings?: string[] | boolean; + omitMessages?: string[]; +} & OptionValues; + +export const buildApiReports = async (opts: Options) => { const tmpDir = cliPaths.resolveTargetRoot( './node_modules/.cache/api-extractor', ); @@ -90,6 +99,7 @@ export default async (opts: OptionValues) => { }); } + console.log(isDocsBuild); if (isDocsBuild) { console.log('# Generating package documentation'); await buildDocs({ @@ -99,37 +109,6 @@ export default async (opts: OptionValues) => { } }; -/** - * Generates the TypeScript declaration files for the specified project, using the provided `tsconfig.json` file. - * - * Any existing declaration files in the `dist-types` directory will be deleted before generating the new ones. - * - * If the `tsc` command exits with a non-zero exit code, the process will be terminated with the same exit code. - * - * @param tsconfigFilePath {string} The path to the `tsconfig.json` file to use for generating the declaration files. - * @returns {Promise} A promise that resolves when the declaration files have been generated. - */ -export async function generateTSC(tsconfigFilePath: string) { - await fs.remove(cliPaths.resolveTargetRoot('dist-types')); - const { status } = spawnSync( - 'yarn', - [ - 'tsc', - ['--project', tsconfigFilePath], - ['--skipLibCheck', 'false'], - ['--incremental', 'false'], - ].flat(), - { - stdio: 'inherit', - shell: true, - cwd: cliPaths.targetRoot, - }, - ); - if (status !== 0) { - process.exit(status || undefined); - } -} - /** * Retrieves the list of package names in the "workspaces" field of the `package.json` file in the current workspace root. * @@ -173,6 +152,6 @@ function parseArrayOption(value: string[] | boolean | undefined) { return value; } return value?.flatMap((str: string) => - str.includes(',') ? str.split(',') : str, + str.includes(',') ? str.split(',').map(s => s.trim()) : str, ); } diff --git a/packages/repo-tools/src/commands/api-reports/generateTSC.ts b/packages/repo-tools/src/commands/api-reports/generateTSC.ts new file mode 100644 index 0000000000..5c10c5085f --- /dev/null +++ b/packages/repo-tools/src/commands/api-reports/generateTSC.ts @@ -0,0 +1,50 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import fs from 'fs-extra'; +import { spawnSync } from 'child_process'; +import { paths as cliPaths } from '../../lib/paths'; + +/** + * Generates the TypeScript declaration files for the specified project, using the provided `tsconfig.json` file. + * + * Any existing declaration files in the `dist-types` directory will be deleted before generating the new ones. + * + * If the `tsc` command exits with a non-zero exit code, the process will be terminated with the same exit code. + * + * @param tsconfigFilePath {string} The path to the `tsconfig.json` file to use for generating the declaration files. + * @returns {Promise} A promise that resolves when the declaration files have been generated. + */ + +export async function generateTSC(tsconfigFilePath: string) { + await fs.remove(cliPaths.resolveTargetRoot('dist-types')); + const { status } = spawnSync( + 'yarn', + [ + 'tsc', + ['--project', tsconfigFilePath], + ['--skipLibCheck', 'false'], + ['--incremental', 'false'], + ].flat(), + { + stdio: 'inherit', + shell: true, + cwd: cliPaths.targetRoot, + }, + ); + if (status !== 0) { + process.exit(status || undefined); + } +} diff --git a/packages/repo-tools/src/commands/index.ts b/packages/repo-tools/src/commands/index.ts index 5742ce7241..3f3b2cdbe7 100644 --- a/packages/repo-tools/src/commands/index.ts +++ b/packages/repo-tools/src/commands/index.ts @@ -39,7 +39,9 @@ export function registerCommands(program: Command) { ) .description('Generate an API report for selected packages') .action( - lazy(() => import('./api-reports/api-reports').then(m => m.default)), + lazy(() => + import('./api-reports/api-reports').then(m => m.buildApiReports), + ), ); program From c981e83612a5a9814d8c52f6bf9071b93778c7de Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 6 Dec 2022 14:32:22 +0100 Subject: [PATCH 072/437] Add highlighting to StackOverflow result list item Signed-off-by: Eric Peterson --- .../search-yet-another-wonderwall-cover.md | 5 ++++ plugins/stack-overflow/api-report.md | 2 ++ plugins/stack-overflow/package.json | 1 + ...ckOverflowSearchResultListItem.stories.tsx | 20 +++++++++++++ ...StackOverflowSearchResultListItem.test.tsx | 24 +++++++++++++++ .../StackOverflowSearchResultListItem.tsx | 29 +++++++++++++++++-- yarn.lock | 1 + 7 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 .changeset/search-yet-another-wonderwall-cover.md diff --git a/.changeset/search-yet-another-wonderwall-cover.md b/.changeset/search-yet-another-wonderwall-cover.md new file mode 100644 index 0000000000..c87f418e50 --- /dev/null +++ b/.changeset/search-yet-another-wonderwall-cover.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-stack-overflow': patch +--- + +The `` component is now able to highlight the result title and/or text when provided. To take advantage of this, pass in the `highlight` prop, similar to how it is done on other result list item components. diff --git a/plugins/stack-overflow/api-report.md b/plugins/stack-overflow/api-report.md index 9af9231094..fe2a970c13 100644 --- a/plugins/stack-overflow/api-report.md +++ b/plugins/stack-overflow/api-report.md @@ -8,6 +8,7 @@ import { BackstagePlugin } from '@backstage/core-plugin-api'; import { CardExtensionProps } from '@backstage/plugin-home'; import { ReactNode } from 'react'; +import { ResultHighlight } from '@backstage/plugin-search-common'; // @public export const HomePageStackOverflowQuestions: ( @@ -45,5 +46,6 @@ export const StackOverflowSearchResultListItem: (props: { result: any; icon?: ReactNode; rank?: number | undefined; + highlight?: ResultHighlight | undefined; }) => JSX.Element; ``` diff --git a/plugins/stack-overflow/package.json b/plugins/stack-overflow/package.json index df1724ac16..2d30f3e2ae 100644 --- a/plugins/stack-overflow/package.json +++ b/plugins/stack-overflow/package.json @@ -27,6 +27,7 @@ "@backstage/core-plugin-api": "workspace:^", "@backstage/plugin-home": "workspace:^", "@backstage/plugin-search-common": "workspace:^", + "@backstage/plugin-search-react": "workspace:^", "@backstage/theme": "workspace:^", "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", diff --git a/plugins/stack-overflow/src/search/StackOverflowSearchResultListItem/StackOverflowSearchResultListItem.stories.tsx b/plugins/stack-overflow/src/search/StackOverflowSearchResultListItem/StackOverflowSearchResultListItem.stories.tsx index df5b8038b0..03d8fdd3d8 100644 --- a/plugins/stack-overflow/src/search/StackOverflowSearchResultListItem/StackOverflowSearchResultListItem.stories.tsx +++ b/plugins/stack-overflow/src/search/StackOverflowSearchResultListItem/StackOverflowSearchResultListItem.stories.tsx @@ -53,3 +53,23 @@ export const WithIcon = () => { /> ); }; + +export const WithHighlight = () => { + return ( + } + highlight={{ + fields: { title: 'Customizing Spotify backstage ui' }, + preTag: '', + postTag: '', + }} + /> + ); +}; diff --git a/plugins/stack-overflow/src/search/StackOverflowSearchResultListItem/StackOverflowSearchResultListItem.test.tsx b/plugins/stack-overflow/src/search/StackOverflowSearchResultListItem/StackOverflowSearchResultListItem.test.tsx index 9e27149efb..bd3640a5ec 100644 --- a/plugins/stack-overflow/src/search/StackOverflowSearchResultListItem/StackOverflowSearchResultListItem.test.tsx +++ b/plugins/stack-overflow/src/search/StackOverflowSearchResultListItem/StackOverflowSearchResultListItem.test.tsx @@ -78,4 +78,28 @@ describe('', () => { value: 1, }); }); + + it('should render highlight', async () => { + await renderInTestApp( + ', + postTag: '', + }} + />, + ); + expect(screen.getByText(/Highlighted Title/i)).toBeInTheDocument(); + expect(screen.getByText(/Highlighted Author/i)).toBeInTheDocument(); + }); }); diff --git a/plugins/stack-overflow/src/search/StackOverflowSearchResultListItem/StackOverflowSearchResultListItem.tsx b/plugins/stack-overflow/src/search/StackOverflowSearchResultListItem/StackOverflowSearchResultListItem.tsx index 6a2c8f5889..9971b0e7e5 100644 --- a/plugins/stack-overflow/src/search/StackOverflowSearchResultListItem/StackOverflowSearchResultListItem.tsx +++ b/plugins/stack-overflow/src/search/StackOverflowSearchResultListItem/StackOverflowSearchResultListItem.tsx @@ -26,17 +26,21 @@ import { Chip, } from '@material-ui/core'; import { useAnalytics } from '@backstage/core-plugin-api'; +import { ResultHighlight } from '@backstage/plugin-search-common'; +import { HighlightedSearchResultText } from '@backstage/plugin-search-react'; type StackOverflowSearchResultListItemProps = { result: any; // TODO(emmaindal): type to StackOverflowDocument. icon?: React.ReactNode; rank?: number; + highlight?: ResultHighlight; }; export const StackOverflowSearchResultListItem = ( props: StackOverflowSearchResultListItemProps, ) => { const { location, title, text, answers, tags } = props.result; + const { highlight } = props; const analytics = useAnalytics(); const handleClick = () => { @@ -55,10 +59,31 @@ export const StackOverflowSearchResultListItem = ( primaryTypographyProps={{ variant: 'h6' }} primary={ - {_unescape(title)} + {highlight?.fields?.title ? ( + + ) : ( + _unescape(title) + )} } - secondary={`Author: ${text}`} + secondary={ + highlight?.fields?.text ? ( + <> + Author:{' '} + + + ) : ( + `Author: ${text}` + ) + } /> {tags && diff --git a/yarn.lock b/yarn.lock index 6311ccc002..3ad6ff25e4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7875,6 +7875,7 @@ __metadata: "@backstage/dev-utils": "workspace:^" "@backstage/plugin-home": "workspace:^" "@backstage/plugin-search-common": "workspace:^" + "@backstage/plugin-search-react": "workspace:^" "@backstage/test-utils": "workspace:^" "@backstage/theme": "workspace:^" "@material-ui/core": ^4.12.2 From a3f4718b9269acd64d92ed4c0354a9174682c667 Mon Sep 17 00:00:00 2001 From: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Date: Tue, 6 Dec 2022 12:27:54 -0600 Subject: [PATCH 073/437] Update .changeset/breezy-apes-mate.md Co-authored-by: Patrik Oldsberg Signed-off-by: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> --- .changeset/breezy-apes-mate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/breezy-apes-mate.md b/.changeset/breezy-apes-mate.md index ac3409bd69..65e7dc2bc4 100644 --- a/.changeset/breezy-apes-mate.md +++ b/.changeset/breezy-apes-mate.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend': patch --- -Adding an optional restriction for locations added through catalog import. Restrictions can be added using the app-config.yaml +Added a new `catalog.rules[].location` configuration that makes it possible to configure catalog rules to only apply to specific locations, either via exact match or a glob pattern. From 3ff751cbbbe4a687925e9b0dc7f6b91d137af963 Mon Sep 17 00:00:00 2001 From: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Date: Tue, 6 Dec 2022 12:28:10 -0600 Subject: [PATCH 074/437] Update plugins/catalog-backend/config.d.ts Co-authored-by: Patrik Oldsberg Signed-off-by: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> --- plugins/catalog-backend/config.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 5da7dd8df9..5f0f8f1ea2 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -61,7 +61,7 @@ export interface Config { exact?: string; /** * The pattern allowed for the location, e.g. - * "https://github.com/org/*\/blob/master/*.yaml. + * "https://github.com/org/*/blob/master/*.yaml. */ pattern?: string; }>; From cf992210ec5775faddd0d8d028f8c6814f1cba8b Mon Sep 17 00:00:00 2001 From: Clare Liguori Date: Tue, 6 Dec 2022 10:50:00 -0800 Subject: [PATCH 075/437] Remove config types from export Signed-off-by: Clare Liguori --- packages/integration-aws-node/api-report.md | 38 --------------------- packages/integration-aws-node/src/index.ts | 7 ---- 2 files changed, 45 deletions(-) diff --git a/packages/integration-aws-node/api-report.md b/packages/integration-aws-node/api-report.md index 58bdde4090..d9ed25e193 100644 --- a/packages/integration-aws-node/api-report.md +++ b/packages/integration-aws-node/api-report.md @@ -26,41 +26,6 @@ export interface AwsCredentialsManager { ): Promise; } -// @public -export type AwsIntegrationAccountConfig = { - accountId: string; - accessKeyId?: string; - secretAccessKey?: string; - profile?: string; - roleName?: string; - partition?: string; - region?: string; - externalId?: string; -}; - -// @public -export type AwsIntegrationConfig = { - accounts: AwsIntegrationAccountConfig[]; - accountDefaults: AwsIntegrationDefaultAccountConfig; - mainAccount: AwsIntegrationMainAccountConfig; -}; - -// @public -export type AwsIntegrationDefaultAccountConfig = { - roleName?: string; - partition?: string; - region?: string; - externalId?: string; -}; - -// @public -export type AwsIntegrationMainAccountConfig = { - accessKeyId?: string; - secretAccessKey?: string; - profile?: string; - region?: string; -}; - // @public export class DefaultAwsCredentialsManager implements AwsCredentialsManager { // (undocumented) @@ -70,8 +35,5 @@ export class DefaultAwsCredentialsManager implements AwsCredentialsManager { ): Promise; } -// @public -export function readAwsIntegrationConfig(config: Config): AwsIntegrationConfig; - // (No @packageDocumentation comment for this package) ``` diff --git a/packages/integration-aws-node/src/index.ts b/packages/integration-aws-node/src/index.ts index 0b6dc630db..30f3e21281 100644 --- a/packages/integration-aws-node/src/index.ts +++ b/packages/integration-aws-node/src/index.ts @@ -14,13 +14,6 @@ * limitations under the License. */ -export { readAwsIntegrationConfig } from './config'; -export type { - AwsIntegrationConfig, - AwsIntegrationAccountConfig, - AwsIntegrationDefaultAccountConfig, - AwsIntegrationMainAccountConfig, -} from './config'; export { DefaultAwsCredentialsManager } from './DefaultAwsCredentialsManager'; export type { AwsCredentialsManager, From ae8234d3918d1298038a526232ecae4926832562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linn=C3=A9a=20Ivansson?= Date: Tue, 6 Dec 2022 20:15:33 +0100 Subject: [PATCH 076/437] Fix prettier and tests. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Linnéa Ivansson --- .../src/components/ErrorReporting/ErrorReporting.tsx | 4 ++-- .../src/components/KubernetesContent.test.tsx | 12 +----------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx b/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx index 6105f29ba8..ed51b7d21d 100644 --- a/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx +++ b/plugins/kubernetes/src/components/ErrorReporting/ErrorReporting.tsx @@ -97,14 +97,14 @@ export const ErrorReporting = ({ detectedErrors }: ErrorReportingProps) => { return ( <> - {errors.length !== 0 && + {errors.length !== 0 && (
- } + )} ); }; diff --git a/plugins/kubernetes/src/components/KubernetesContent.test.tsx b/plugins/kubernetes/src/components/KubernetesContent.test.tsx index 231eb3b37a..ffa5358537 100644 --- a/plugins/kubernetes/src/components/KubernetesContent.test.tsx +++ b/plugins/kubernetes/src/components/KubernetesContent.test.tsx @@ -45,11 +45,6 @@ describe('KubernetesContent', () => { />, ), ); - - expect(getByText('Error Reporting')).toBeInTheDocument(); - expect( - getByText('Nice! There are no errors to report!'), - ).toBeInTheDocument(); expect(getByText('Your Clusters')).toBeInTheDocument(); // TODO add a prompt for the user to configure their clusters }); @@ -94,9 +89,6 @@ describe('KubernetesContent', () => { ), ); - expect( - getByText('Nice! There are no errors to report!'), - ).toBeInTheDocument(); expect(getByText('cluster-1')).toBeInTheDocument(); expect(getByText('Cluster')).toBeInTheDocument(); expect(getByText('10 pods')).toBeInTheDocument(); @@ -148,7 +140,7 @@ describe('KubernetesContent', () => { }, error: undefined, }); - const { getByText, getAllByText, queryByText } = render( + const { getByText, getAllByText } = render( wrapInTestApp( { />, ), ); - - expect(queryByText('Nice! There are no errors to report!')).toBeNull(); expect(getAllByText('Cluster')).toHaveLength(2); expect(getByText('cluster-a')).toBeInTheDocument(); expect(getByText('10 pods')).toBeInTheDocument(); From 4e65bd9a58ecae8390d43f7f28692ece070487f4 Mon Sep 17 00:00:00 2001 From: Sarah Medeiros Date: Tue, 6 Dec 2022 17:56:24 -0500 Subject: [PATCH 077/437] Fix owner and lifecycle picker Signed-off-by: Sarah Medeiros --- .../EntityLifecyclePicker.tsx | 22 +++++++------- .../EntityOwnerPicker/EntityOwnerPicker.tsx | 30 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx index f6e6b4eeaa..693c8ab85c 100644 --- a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx +++ b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx @@ -83,17 +83,17 @@ export const EntityLifecyclePicker = () => { }); }, [selectedLifecycles, updateFilters]); - const availableLifecycles = useMemo( - () => - [ - ...new Set( - backendEntities - .map((e: Entity) => e.spec?.lifecycle) - .filter(Boolean) as string[], - ), - ].sort(), - [backendEntities], - ); + const availableLifecycles = useMemo(() => { + const lifecycles = [ + ...new Set( + backendEntities + .map((e: Entity) => e.spec?.lifecycle) + .filter(Boolean) as string[], + ), + ].sort(); + if (lifecycles.length === 0) setSelectedLifecycles([]); + return lifecycles; + }, [backendEntities]); if (!availableLifecycles.length) return null; diff --git a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx index 0a4a79857e..d389ead06a 100644 --- a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx +++ b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx @@ -83,21 +83,21 @@ export const EntityOwnerPicker = () => { }); }, [selectedOwners, updateFilters]); - const availableOwners = useMemo( - () => - [ - ...new Set( - backendEntities - .flatMap((e: Entity) => - getEntityRelations(e, RELATION_OWNED_BY).map(o => - humanizeEntityRef(o, { defaultKind: 'group' }), - ), - ) - .filter(Boolean) as string[], - ), - ].sort(), - [backendEntities], - ); + const availableOwners = useMemo(() => { + const owners = [ + ...new Set( + backendEntities + .flatMap((e: Entity) => + getEntityRelations(e, RELATION_OWNED_BY).map(o => + humanizeEntityRef(o, { defaultKind: 'group' }), + ), + ) + .filter(Boolean) as string[], + ), + ].sort(); + if (owners.length === 0) setSelectedOwners([]); + return owners; + }, [backendEntities]); if (!availableOwners.length) return null; From e5f61559d5a9e90e210c4d0635948e6e2cb09b96 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 7 Dec 2022 08:00:47 +0000 Subject: [PATCH 078/437] use nunjucks "square bracket syntax" for steps in docs Signed-off-by: Brian Fletcher --- docs/features/software-catalog/descriptor-format.md | 2 +- docs/features/software-templates/adding-templates.md | 2 +- .../migrating-from-v1beta2-to-v1beta3.md | 10 +++++----- docs/features/software-templates/writing-templates.md | 10 +++++----- .../default-app/examples/template/template.yaml | 6 +++--- .../sample-templates/bitbucket-demo/template.yaml | 6 +++--- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 27b26659cf..be71c9d174 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -703,7 +703,7 @@ spec: name: Register action: catalog:register input: - repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + repoContentsUrl: '{{ steps["publish"].output.repoContentsUrl }}' catalogInfoPath: '/catalog-info.yaml' ``` diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index e8176a688b..e11dad2acb 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -76,7 +76,7 @@ spec: name: Register action: catalog:register input: - repoContentsUrl: ${{ steps.publish.output.repoContentsUrl }} + repoContentsUrl: ${{ steps["publish"].output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' ``` diff --git a/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md b/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md index a2277e096c..c36f02a3df 100644 --- a/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md +++ b/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md @@ -169,14 +169,14 @@ These should be moved to `links` under the `output` object instead. ```diff output: -- remoteUrl: '{{ steps.publish.output.remoteUrl }}' -- entityRef: '{{ steps.register.output.entityRef }}' +- remoteUrl: '{{ steps["publish"].output.remoteUrl }}' +- entityRef: '{{ steps["register"].output.entityRef }}' + links: + - title: Repository -+ url: ${{ steps.publish.output.remoteUrl }} ++ url: ${{ steps["publish"].output.remoteUrl }} + - title: Open in catalog + icon: catalog -+ entityRef: ${{ steps.register.output.entityRef }} ++ entityRef: ${{ steps["register"].output.entityRef }} ``` @@ -206,7 +206,7 @@ Alternatively, it's possible to keep the `dash-case` syntax and use brackets for ```yaml input: - repoUrl: ${{ steps['my-custom-action'].output.repoUrl }} + repoUrl: ${{ steps["my-custom-action"].output.repoUrl }} ``` ### Summary diff --git a/docs/features/software-templates/writing-templates.md b/docs/features/software-templates/writing-templates.md index 826121bd02..d53ce3716c 100644 --- a/docs/features/software-templates/writing-templates.md +++ b/docs/features/software-templates/writing-templates.md @@ -88,17 +88,17 @@ spec: name: Register action: catalog:register input: - repoContentsUrl: ${{ steps.publish.output.repoContentsUrl }} + repoContentsUrl: ${{ steps["publish"].output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' # some outputs which are saved along with the job for use in the frontend output: links: - title: Repository - url: ${{ steps.publish.output.remoteUrl }} + url: ${{ steps["publish"].output.remoteUrl }} - title: Open in catalog icon: catalog - entityRef: ${{ steps.register.output.entityRef }} + entityRef: ${{ steps["register"].output.entityRef }} ``` Let's dive in and pick apart what each of these sections do and what they are. @@ -505,10 +505,10 @@ The main two that are used are the following: output: links: - title: Repository - url: ${{ steps.publish.output.remoteUrl }} # link to the remote repository + url: ${{ steps["publish"].output.remoteUrl }} # link to the remote repository - title: Open in catalog icon: catalog - entityRef: ${{ steps.register.output.entityRef }} # link to the entity that has been ingested to the catalog + entityRef: ${{ steps["register"].output.entityRef }} # link to the entity that has been ingested to the catalog ``` ## The templating syntax diff --git a/packages/create-app/templates/default-app/examples/template/template.yaml b/packages/create-app/templates/default-app/examples/template/template.yaml index 50052b7a7c..39c5d2326f 100644 --- a/packages/create-app/templates/default-app/examples/template/template.yaml +++ b/packages/create-app/templates/default-app/examples/template/template.yaml @@ -61,14 +61,14 @@ spec: name: Register action: catalog:register input: - repoContentsUrl: ${{ steps.publish.output.repoContentsUrl }} + repoContentsUrl: ${{ steps["publish"].output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' # Outputs are displayed to the user after a successful execution of the template. output: links: - title: Repository - url: ${{ steps.publish.output.remoteUrl }} + url: ${{ steps["publish"].output.remoteUrl }} - title: Open in catalog icon: catalog - entityRef: ${{ steps.register.output.entityRef }} + entityRef: ${{ steps["register"].output.entityRef }} diff --git a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml index 52714f6d73..8fd60b594a 100644 --- a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml @@ -68,13 +68,13 @@ spec: name: Register action: catalog:register input: - repoContentsUrl: ${{ steps.publish.output.repoContentsUrl }} + repoContentsUrl: ${{ steps["publish"].output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' output: links: - title: Repository - url: ${{ steps.publish.output.remoteUrl }} + url: ${{ steps["publish"].output.remoteUrl }} - title: Open in catalog icon: catalog - entityRef: ${{ steps.register.output.entityRef }} + entityRef: ${{ steps["register"].output.entityRef }} From 935b66a646bf16338e4cf510414493e7a775ba2d Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 7 Dec 2022 08:10:42 +0000 Subject: [PATCH 079/437] add changeset Signed-off-by: Brian Fletcher --- .changeset/dirty-ads-refuse.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/dirty-ads-refuse.md diff --git a/.changeset/dirty-ads-refuse.md b/.changeset/dirty-ads-refuse.md new file mode 100644 index 0000000000..9c982199e7 --- /dev/null +++ b/.changeset/dirty-ads-refuse.md @@ -0,0 +1,6 @@ +--- +'@backstage/create-app': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Change step output template examples to use square bracket syntax. From 808b53d492679d5e4e83932e6397438c56cb5f55 Mon Sep 17 00:00:00 2001 From: Juan Pablo Garcia Ripa Date: Wed, 7 Dec 2022 11:43:21 +0100 Subject: [PATCH 080/437] change the options to only allow csv Signed-off-by: Juan Pablo Garcia Ripa --- .changeset/lemon-coats-camp.md | 7 +- packages/repo-tools/cli-report.md | 10 +- .../commands/api-reports/api-reports.test.ts | 194 +++++++----------- .../src/commands/api-reports/api-reports.ts | 58 +++--- ...rateTSC.ts => generateTypeDeclarations.ts} | 2 +- packages/repo-tools/src/commands/index.ts | 14 +- 6 files changed, 120 insertions(+), 165 deletions(-) rename packages/repo-tools/src/commands/api-reports/{generateTSC.ts => generateTypeDeclarations.ts} (95%) diff --git a/.changeset/lemon-coats-camp.md b/.changeset/lemon-coats-camp.md index ae31098e2f..f30a20f432 100644 --- a/.changeset/lemon-coats-camp.md +++ b/.changeset/lemon-coats-camp.md @@ -1,12 +1,11 @@ --- -'@backstage/repo-tools': minor +'@backstage/repo-tools': patch --- Add new command options to the `api-report` -- added `--allow-warnings`, `-a` to continue processing packages if some packages have warnings +- added `--allow-warnings`, `-a` to continue processing packages if selected packages have warnings +- added `--allow-all-warnings` to continue processing packages any packages have warnings - added `--omit-messages`, `-o` to pass some warnings messages code to be omitted from the api-report.md files -- added `--paths`, `-p` to select packages path to process - The `paths` argument for this command now takes as default the value on `workspaces.packages` inside the root package.json -- Removed the `paths` argument replaced by the option `--paths` - change the path resolution to use the `@backstage/cli-common` packages instead diff --git a/packages/repo-tools/cli-report.md b/packages/repo-tools/cli-report.md index d3446e0262..3378d20ea7 100644 --- a/packages/repo-tools/cli-report.md +++ b/packages/repo-tools/cli-report.md @@ -12,7 +12,7 @@ Options: -h, --help Commands: - api-reports [options] + api-reports [options] [paths...] type-deps help [command] ``` @@ -20,15 +20,15 @@ Commands: ### `backstage-repo-tools api-reports` ``` -Usage: backstage-repo-tools api-reports [options] +Usage: backstage-repo-tools api-reports [options] [paths...] Options: - -p --paths [paths...] --ci --tsc --docs - -a, --allow-warnings [allowWarningsPaths...] - -o, --omit-messages + -a, --allow-warnings + --allow-all-warnings + -o, --omit-messages -h, --help ``` diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.test.ts b/packages/repo-tools/src/commands/api-reports/api-reports.test.ts index fa5bfd6973..7121d8a06f 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.test.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.test.ts @@ -26,9 +26,9 @@ import { } from './api-extractor'; import { buildApiReports } from './api-reports'; -import { generateTSC } from './generateTSC'; +import { generateTypeDeclarations } from './generateTypeDeclarations'; -jest.mock('./generateTSC'); +jest.mock('./generateTypeDeclarations'); // create mocks for the dependencies of the `buildApiReports` function jest.mock('./api-extractor', () => ({ createTemporaryTsConfig: jest.fn(), @@ -44,17 +44,17 @@ jest.mock('./api-extractor', () => ({ buildDocs: jest.fn(), })); -const paths = pathsLib.paths; +const projectPaths = pathsLib.paths; -jest.spyOn(paths, 'targetRoot', 'get').mockReturnValue('/root'); -jest.spyOn(paths, 'resolveTargetRoot').mockImplementation((...path) => { +jest.spyOn(projectPaths, 'targetRoot', 'get').mockReturnValue('/root'); +jest.spyOn(projectPaths, 'resolveTargetRoot').mockImplementation((...path) => { return resolvePath('/root', ...path); }); describe('buildApiReports', () => { beforeEach(() => { mockFs({ - [paths.targetRoot]: { + [projectPaths.targetRoot]: { 'package.json': JSON.stringify({ workspaces: { packages: ['packages/*', 'plugins/*'] }, }), @@ -89,8 +89,9 @@ describe('buildApiReports', () => { }); it('should run whitout any options', async () => { const opts = {}; + const paths: string[] = []; - await buildApiReports(opts); + await buildApiReports(paths, opts); expect(categorizePackageDirs).toHaveBeenCalledWith([ 'packages/package-a', @@ -100,7 +101,7 @@ describe('buildApiReports', () => { 'plugins/plugin-c', ]); - expect(generateTSC).not.toHaveBeenCalled(); + expect(generateTypeDeclarations).not.toHaveBeenCalled(); expect(runApiExtraction).toHaveBeenCalledWith({ packageDirs: [ 'packages/package-a', @@ -110,7 +111,7 @@ describe('buildApiReports', () => { 'plugins/plugin-c', ], tsconfigFilePath: '/root/tsconfig.json', - allowWarnings: undefined, + allowWarnings: [], omitMessages: [], isLocalBuild: true, outputDir: '/root/node_modules/.cache/api-extractor', @@ -132,11 +133,10 @@ describe('buildApiReports', () => { describe('paths', () => { it('should generate API reports for one specific package', async () => { - const opts = { - paths: ['packages/package-a'], - }; + const paths = ['packages/package-a']; + const opts = {}; - await buildApiReports(opts); + await buildApiReports(paths, opts); expect(categorizePackageDirs).toHaveBeenCalledWith([ 'packages/package-a', @@ -145,7 +145,7 @@ describe('buildApiReports', () => { expect(runApiExtraction).toHaveBeenCalledWith({ packageDirs: ['packages/package-a'], tsconfigFilePath: '/root/tsconfig.json', - allowWarnings: undefined, + allowWarnings: [], omitMessages: [], isLocalBuild: true, outputDir: '/root/node_modules/.cache/api-extractor', @@ -158,11 +158,10 @@ describe('buildApiReports', () => { expect(buildDocs).not.toHaveBeenCalled(); }); it('should generate API reports for multiple specific packages', async () => { - const opts = { - paths: ['packages/package-a', 'packages/package-b'], - }; + const paths = ['packages/package-a', 'packages/package-b']; + const opts = {}; - await buildApiReports(opts); + await buildApiReports(paths, opts); expect(categorizePackageDirs).toHaveBeenCalledWith([ 'packages/package-a', @@ -172,7 +171,7 @@ describe('buildApiReports', () => { expect(runApiExtraction).toHaveBeenCalledWith({ packageDirs: ['packages/package-a', 'packages/package-b'], tsconfigFilePath: '/root/tsconfig.json', - allowWarnings: undefined, + allowWarnings: [], omitMessages: [], isLocalBuild: true, outputDir: '/root/node_modules/.cache/api-extractor', @@ -185,11 +184,10 @@ describe('buildApiReports', () => { expect(buildDocs).not.toHaveBeenCalled(); }); it('should generate API reports for all packages matching the glob pattern', async () => { - const opts = { - paths: ['packages/*'], - }; + const paths = ['packages/*']; + const opts = {}; - await buildApiReports(opts); + await buildApiReports(paths, opts); expect(categorizePackageDirs).toHaveBeenCalledWith([ 'packages/package-a', @@ -199,7 +197,7 @@ describe('buildApiReports', () => { expect(runApiExtraction).toHaveBeenCalledWith({ packageDirs: ['packages/package-a', 'packages/package-b'], tsconfigFilePath: '/root/tsconfig.json', - allowWarnings: undefined, + allowWarnings: [], omitMessages: [], isLocalBuild: true, outputDir: '/root/node_modules/.cache/api-extractor', @@ -213,11 +211,10 @@ describe('buildApiReports', () => { }); it('should generate API reports for all packages matching multiple glob patterns', async () => { - const opts = { - paths: ['packages/*', 'plugins/*a'], - }; + const paths = ['packages/*', 'plugins/*a']; + const opts = {}; - await buildApiReports(opts); + await buildApiReports(paths, opts); expect(categorizePackageDirs).toHaveBeenCalledWith([ 'packages/package-a', @@ -232,7 +229,7 @@ describe('buildApiReports', () => { 'plugins/plugin-a', ], tsconfigFilePath: '/root/tsconfig.json', - allowWarnings: undefined, + allowWarnings: [], omitMessages: [], isLocalBuild: true, outputDir: '/root/node_modules/.cache/api-extractor', @@ -250,11 +247,10 @@ describe('buildApiReports', () => { }); it('should generate API reports for specific packages and glob pattern', async () => { - const opts = { - paths: ['packages/package-a', 'plugins/*'], - }; + const opts = {}; + const paths = ['packages/package-a', 'plugins/*']; - await buildApiReports(opts); + await buildApiReports(paths, opts); expect(categorizePackageDirs).toHaveBeenCalledWith([ 'packages/package-a', @@ -271,7 +267,7 @@ describe('buildApiReports', () => { 'plugins/plugin-c', ], tsconfigFilePath: '/root/tsconfig.json', - allowWarnings: undefined, + allowWarnings: [], omitMessages: [], isLocalBuild: true, outputDir: '/root/node_modules/.cache/api-extractor', @@ -290,31 +286,13 @@ describe('buildApiReports', () => { }); }); describe('allowWarnings', () => { - it('should accept boolean values', async () => { - const opts = { - paths: ['packages/*'], - allowWarnings: true, - }; - - await buildApiReports(opts); - - expect(runApiExtraction).toHaveBeenCalledWith({ - packageDirs: ['packages/package-a', 'packages/package-b'], - tsconfigFilePath: '/root/tsconfig.json', - allowWarnings: true, - omitMessages: [], - isLocalBuild: true, - outputDir: '/root/node_modules/.cache/api-extractor', - }); - }); - it('should accept single path value', async () => { const opts = { - paths: ['packages/*'], - allowWarnings: ['packages/package-a'], + allowWarnings: 'packages/package-a', }; + const paths = ['packages/*']; - await buildApiReports(opts); + await buildApiReports(paths, opts); expect(runApiExtraction).toHaveBeenCalledWith({ packageDirs: ['packages/package-a', 'packages/package-b'], @@ -326,31 +304,13 @@ describe('buildApiReports', () => { }); }); - it('should accept multiple path values as array', async () => { - const opts = { - paths: ['packages/*'], - allowWarnings: ['packages/package-a', 'packages/package-b'], - }; - - await buildApiReports(opts); - - expect(runApiExtraction).toHaveBeenCalledWith({ - packageDirs: ['packages/package-a', 'packages/package-b'], - tsconfigFilePath: '/root/tsconfig.json', - allowWarnings: ['packages/package-a', 'packages/package-b'], - omitMessages: [], - isLocalBuild: true, - outputDir: '/root/node_modules/.cache/api-extractor', - }); - }); - it('should accept multiple path values as comma separated string', async () => { const opts = { - paths: ['packages/*'], - allowWarnings: ['packages/package-a,packages/package-b'], + allowWarnings: 'packages/package-a,packages/package-b', }; + const paths = ['packages/*']; - await buildApiReports(opts); + await buildApiReports(paths, opts); expect(runApiExtraction).toHaveBeenCalledWith({ packageDirs: ['packages/package-a', 'packages/package-b'], @@ -364,11 +324,11 @@ describe('buildApiReports', () => { it('should accept multiple path values as comma separated string with spaces', async () => { const opts = { - paths: ['packages/*'], - allowWarnings: ['packages/package-a, packages/package-b'], + allowWarnings: 'packages/package-a, packages/package-b', }; + const paths = ['packages/*']; - await buildApiReports(opts); + await buildApiReports(paths, opts); expect(runApiExtraction).toHaveBeenCalledWith({ packageDirs: ['packages/package-a', 'packages/package-b'], @@ -380,54 +340,56 @@ describe('buildApiReports', () => { }); }); }); + describe('allowAllWarnings', () => { + it('should accept boolean values', async () => { + const opts = { + allowAllWarnings: true, + }; + const paths = ['packages/*']; + + await buildApiReports(paths, opts); + + expect(runApiExtraction).toHaveBeenCalledWith({ + packageDirs: ['packages/package-a', 'packages/package-b'], + tsconfigFilePath: '/root/tsconfig.json', + allowWarnings: true, + omitMessages: [], + isLocalBuild: true, + outputDir: '/root/node_modules/.cache/api-extractor', + }); + }); + }); describe('omitMessages', () => { it('should accept single message value', async () => { const opts = { - paths: ['packages/*'], - omitMessages: ['ae-missing-release-tag'], + omitMessages: 'ae-missing-release-tag', }; + const paths = ['packages/*']; - await buildApiReports(opts); + await buildApiReports(paths, opts); expect(runApiExtraction).toHaveBeenCalledWith({ packageDirs: ['packages/package-a', 'packages/package-b'], tsconfigFilePath: '/root/tsconfig.json', - allowWarnings: undefined, + allowWarnings: [], omitMessages: ['ae-missing-release-tag'], isLocalBuild: true, outputDir: '/root/node_modules/.cache/api-extractor', }); }); - it('should accept multiple message values as array', async () => { - const opts = { - paths: ['packages/*'], - omitMessages: ['ae-missing-release-tag', 'ae-missing-annotations'], - }; - - await buildApiReports(opts); - - expect(runApiExtraction).toHaveBeenCalledWith({ - packageDirs: ['packages/package-a', 'packages/package-b'], - tsconfigFilePath: '/root/tsconfig.json', - allowWarnings: undefined, - omitMessages: ['ae-missing-release-tag', 'ae-missing-annotations'], - isLocalBuild: true, - outputDir: '/root/node_modules/.cache/api-extractor', - }); - }); it('should accept multiple message values as comma separated string', async () => { const opts = { - paths: ['packages/*'], - omitMessages: ['ae-missing-release-tag,ae-missing-annotations'], + omitMessages: 'ae-missing-release-tag,ae-missing-annotations', }; + const paths = ['packages/*']; - await buildApiReports(opts); + await buildApiReports(paths, opts); expect(runApiExtraction).toHaveBeenCalledWith({ packageDirs: ['packages/package-a', 'packages/package-b'], tsconfigFilePath: '/root/tsconfig.json', - allowWarnings: undefined, + allowWarnings: [], omitMessages: ['ae-missing-release-tag', 'ae-missing-annotations'], isLocalBuild: true, outputDir: '/root/node_modules/.cache/api-extractor', @@ -436,16 +398,16 @@ describe('buildApiReports', () => { it('should accept multiple message values as comma separated string with spaces', async () => { const opts = { - paths: ['packages/*'], - omitMessages: ['ae-missing-release-tag, ae-missing-annotations'], + omitMessages: 'ae-missing-release-tag, ae-missing-annotations', }; + const paths = ['packages/*']; - await buildApiReports(opts); + await buildApiReports(paths, opts); expect(runApiExtraction).toHaveBeenCalledWith({ packageDirs: ['packages/package-a', 'packages/package-b'], tsconfigFilePath: '/root/tsconfig.json', - allowWarnings: undefined, + allowWarnings: [], omitMessages: ['ae-missing-release-tag', 'ae-missing-annotations'], isLocalBuild: true, outputDir: '/root/node_modules/.cache/api-extractor', @@ -455,16 +417,16 @@ describe('buildApiReports', () => { describe('isCI', () => { it('should set localBuild to false if CI option is passed', async () => { const opts = { - paths: ['packages/*'], ci: true, }; + const paths = ['packages/*']; - await buildApiReports(opts); + await buildApiReports(paths, opts); expect(runApiExtraction).toHaveBeenCalledWith({ packageDirs: ['packages/package-a', 'packages/package-b'], tsconfigFilePath: '/root/tsconfig.json', - allowWarnings: undefined, + allowWarnings: [], omitMessages: [], isLocalBuild: false, outputDir: '/root/node_modules/.cache/api-extractor', @@ -478,11 +440,11 @@ describe('buildApiReports', () => { describe('docs', () => { it('should run typedoc if docs option is passed', async () => { const opts = { - paths: ['packages/*'], docs: true, }; + const paths = ['packages/*']; - await buildApiReports(opts); + await buildApiReports(paths, opts); expect(buildDocs).toHaveBeenCalledWith({ inputDir: '/root/node_modules/.cache/api-extractor', @@ -493,13 +455,13 @@ describe('buildApiReports', () => { describe('tsc', () => { it('should run tsc if tsc option is passed', async () => { const opts = { - paths: ['packages/*'], tsc: true, }; + const paths = ['packages/*']; - await buildApiReports(opts); + await buildApiReports(paths, opts); - expect(generateTSC).toHaveBeenCalled(); + expect(generateTypeDeclarations).toHaveBeenCalled(); }); }); }); diff --git a/packages/repo-tools/src/commands/api-reports/api-reports.ts b/packages/repo-tools/src/commands/api-reports/api-reports.ts index b59f498d7b..6270072912 100644 --- a/packages/repo-tools/src/commands/api-reports/api-reports.ts +++ b/packages/repo-tools/src/commands/api-reports/api-reports.ts @@ -24,18 +24,18 @@ import { buildDocs, } from './api-extractor'; import { findPackageDirs, paths as cliPaths } from '../../lib/paths'; -import { generateTSC } from './generateTSC'; +import { generateTypeDeclarations } from './generateTypeDeclarations'; type Options = { ci?: boolean; docs?: boolean; tsc?: boolean; - paths?: string[]; - allowWarnings?: string[] | boolean; - omitMessages?: string[]; + allowWarnings?: string; + allowAllWarnings?: boolean; + omitMessages?: string; } & OptionValues; -export const buildApiReports = async (opts: Options) => { +export const buildApiReports = async (paths: string[] = [], opts: Options) => { const tmpDir = cliPaths.resolveTargetRoot( './node_modules/.cache/api-extractor', ); @@ -43,15 +43,16 @@ export const buildApiReports = async (opts: Options) => { const isCiBuild = opts.ci; const isDocsBuild = opts.docs; const runTsc = opts.tsc; - - const parsedPaths = parseArrayOption(opts.paths); - const isAllPackages = !Array.isArray(parsedPaths) || !parsedPaths?.length; - const selectedPaths = isAllPackages ? await getWorkspacePkgs() : parsedPaths; - const selectedPackageDirs = await findPackageDirs(selectedPaths); - const allowWarnings = parseArrayOption(opts.allowWarnings); + const allowAllWarnings = opts.allowAllWarnings; const omitMessages = parseArrayOption(opts.omitMessages); + const isAllPackages = !paths?.length; + const selectedPaths = isAllPackages + ? await getWorkspacePackagePathPatterns() + : paths; + const selectedPackageDirs = await findPackageDirs(selectedPaths); + if (isAllPackages && !isCiBuild && !isDocsBuild) { console.log(''); console.log( @@ -59,7 +60,7 @@ export const buildApiReports = async (opts: Options) => { ); console.log(''); console.log( - ' yarn build:api-reports -p packages/config -p packages/core-plugin-api,plugins/*', + ' yarn build:api-reports packages/config packages/core-plugin-api plugins/*', ); console.log(''); } @@ -73,7 +74,7 @@ export const buildApiReports = async (opts: Options) => { if (runTsc) { console.log('# Compiling TypeScript'); - await generateTSC(tsconfigFilePath); + await generateTypeDeclarations(tsconfigFilePath); } const { tsPackageDirs, cliPackageDirs } = await categorizePackageDirs( @@ -87,7 +88,7 @@ export const buildApiReports = async (opts: Options) => { outputDir: tmpDir, isLocalBuild: !isCiBuild, tsconfigFilePath, - allowWarnings, + allowWarnings: allowAllWarnings || allowWarnings, omitMessages: Array.isArray(omitMessages) ? omitMessages : [], }); } @@ -99,7 +100,6 @@ export const buildApiReports = async (opts: Options) => { }); } - console.log(isDocsBuild); if (isDocsBuild) { console.log('# Generating package documentation'); await buildDocs({ @@ -116,7 +116,7 @@ export const buildApiReports = async (opts: Options) => { * * @returns {Promise} The list of package names, or `undefined` if not found. */ -async function getWorkspacePkgs() { +async function getWorkspacePackagePathPatterns() { const pkgJson = await fs .readJson(cliPaths.resolveTargetRoot('package.json')) .catch(error => { @@ -130,28 +130,22 @@ async function getWorkspacePkgs() { } /** - * Splits each string in the input array on comma, and returns an array of the resulting substrings. - * If the input array is `undefined`, returns `undefined`. If the input value is `true` or `false`, - * returns the value as-is. + * Splits the input string on comma, and returns an array of the resulting substrings. + * for `undefined` or an empty string, returns an empty array. * - * @param value An array of strings to be split on comma, or a boolean value (inherithed from commanderjs array args). - * @returns An array of the resulting substrings, the original boolean value, or `undefined` if the input value is `undefined`. + * @param value A string to be split on comma. + * @returns An array of the resulting substrings, or an empty array if the input value is `undefined` or an empty string. * * @example - * parseOption(['foo,bar,baz']) + * parseOption('foo,bar,baz') * // returns ['foo', 'bar', 'baz'] * - * parseOption(true) - * // returns true + * parseOption('') + * // returns [] * * parseOption() - * // returns undefined + * // returns [] */ -function parseArrayOption(value: string[] | boolean | undefined) { - if (typeof value === 'boolean') { - return value; - } - return value?.flatMap((str: string) => - str.includes(',') ? str.split(',').map(s => s.trim()) : str, - ); +function parseArrayOption(value: string | undefined) { + return value ? value.split(',').map(s => s.trim()) : []; } diff --git a/packages/repo-tools/src/commands/api-reports/generateTSC.ts b/packages/repo-tools/src/commands/api-reports/generateTypeDeclarations.ts similarity index 95% rename from packages/repo-tools/src/commands/api-reports/generateTSC.ts rename to packages/repo-tools/src/commands/api-reports/generateTypeDeclarations.ts index 5c10c5085f..58f4117bc2 100644 --- a/packages/repo-tools/src/commands/api-reports/generateTSC.ts +++ b/packages/repo-tools/src/commands/api-reports/generateTypeDeclarations.ts @@ -28,7 +28,7 @@ import { paths as cliPaths } from '../../lib/paths'; * @returns {Promise} A promise that resolves when the declaration files have been generated. */ -export async function generateTSC(tsconfigFilePath: string) { +export async function generateTypeDeclarations(tsconfigFilePath: string) { await fs.remove(cliPaths.resolveTargetRoot('dist-types')); const { status } = spawnSync( 'yarn', diff --git a/packages/repo-tools/src/commands/index.ts b/packages/repo-tools/src/commands/index.ts index 3f3b2cdbe7..f7a16b6d4e 100644 --- a/packages/repo-tools/src/commands/index.ts +++ b/packages/repo-tools/src/commands/index.ts @@ -20,21 +20,21 @@ import { exitWithError } from '../lib/errors'; export function registerCommands(program: Command) { program - .command('api-reports') - .option( - '-p --paths [paths...]', - 'paths of package folder to extract API reports, `workspaces.packages` from root packages.json by default. Allows glob patterns and comma separated values', - ) + .command('api-reports [paths...]') .option('--ci', 'CI run checks that there is no changes on API reports') .option('--tsc', 'executes the tsc compilation before extracting the APIs') .option('--docs', 'generates the api documentation') .option( - '-a, --allow-warnings [allowWarningsPaths...]', + '-a, --allow-warnings ', 'continue processing packages after getting errors on selected packages Allows glob patterns and comma separated values (i.e. packages/core,plugins/core-*)', + ) + .option( + '--allow-all-warnings', + 'continue processing packages after getting errors on all packages', false, ) .option( - '-o, --omit-messages ', + '-o, --omit-messages ', 'select some message code to be omited on the API Extractor (comma separated values i.e ae-cyclic-inherit-doc,ae-missing-getter )', ) .description('Generate an API report for selected packages') From 626a71fa06571f8808b1468a9ae930fc8230b0e3 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 7 Dec 2022 13:03:27 +0000 Subject: [PATCH 081/437] fix review comments Signed-off-by: Brian Fletcher --- docs/features/software-catalog/descriptor-format.md | 2 +- docs/features/software-templates/adding-templates.md | 2 +- .../migrating-from-v1beta2-to-v1beta3.md | 10 +++++----- docs/features/software-templates/writing-templates.md | 10 +++++----- .../default-app/examples/template/template.yaml | 2 +- .../scaffolder-backend-module-cookiecutter/README.md | 6 +++--- plugins/scaffolder-backend-module-rails/README.md | 6 +++--- plugins/scaffolder-backend-module-yeoman/README.md | 6 +++--- .../sample-templates/bitbucket-demo/template.yaml | 4 ++-- 9 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index be71c9d174..6596e65464 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -703,7 +703,7 @@ spec: name: Register action: catalog:register input: - repoContentsUrl: '{{ steps["publish"].output.repoContentsUrl }}' + repoContentsUrl: {{ steps['publish'].output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' ``` diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index e11dad2acb..6b0e2330a4 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -76,7 +76,7 @@ spec: name: Register action: catalog:register input: - repoContentsUrl: ${{ steps["publish"].output.repoContentsUrl }} + repoContentsUrl: ${{ steps['publish'].output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' ``` diff --git a/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md b/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md index c36f02a3df..6e37a4df3e 100644 --- a/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md +++ b/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md @@ -169,14 +169,14 @@ These should be moved to `links` under the `output` object instead. ```diff output: -- remoteUrl: '{{ steps["publish"].output.remoteUrl }}' -- entityRef: '{{ steps["register"].output.entityRef }}' +- remoteUrl: {{ steps['publish'].output.remoteUrl }} +- entityRef: {{ steps['register'].output.entityRef }} + links: + - title: Repository -+ url: ${{ steps["publish"].output.remoteUrl }} ++ url: ${{ steps['publish'].output.remoteUrl }} + - title: Open in catalog + icon: catalog -+ entityRef: ${{ steps["register"].output.entityRef }} ++ entityRef: ${{ steps['register'].output.entityRef }} ``` @@ -206,7 +206,7 @@ Alternatively, it's possible to keep the `dash-case` syntax and use brackets for ```yaml input: - repoUrl: ${{ steps["my-custom-action"].output.repoUrl }} + repoUrl: ${{ steps['my-custom-action'].output.repoUrl }} ``` ### Summary diff --git a/docs/features/software-templates/writing-templates.md b/docs/features/software-templates/writing-templates.md index d53ce3716c..e5fdb034fb 100644 --- a/docs/features/software-templates/writing-templates.md +++ b/docs/features/software-templates/writing-templates.md @@ -88,17 +88,17 @@ spec: name: Register action: catalog:register input: - repoContentsUrl: ${{ steps["publish"].output.repoContentsUrl }} + repoContentsUrl: ${{ steps['publish'].output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' # some outputs which are saved along with the job for use in the frontend output: links: - title: Repository - url: ${{ steps["publish"].output.remoteUrl }} + url: ${{ steps['publish'].output.remoteUrl }} - title: Open in catalog icon: catalog - entityRef: ${{ steps["register"].output.entityRef }} + entityRef: ${{ steps['register'].output.entityRef }} ``` Let's dive in and pick apart what each of these sections do and what they are. @@ -505,10 +505,10 @@ The main two that are used are the following: output: links: - title: Repository - url: ${{ steps["publish"].output.remoteUrl }} # link to the remote repository + url: ${{ steps['publish'].output.remoteUrl }} # link to the remote repository - title: Open in catalog icon: catalog - entityRef: ${{ steps["register"].output.entityRef }} # link to the entity that has been ingested to the catalog + entityRef: ${{ steps['register'].output.entityRef }} # link to the entity that has been ingested to the catalog ``` ## The templating syntax diff --git a/packages/create-app/templates/default-app/examples/template/template.yaml b/packages/create-app/templates/default-app/examples/template/template.yaml index 39c5d2326f..be3533162f 100644 --- a/packages/create-app/templates/default-app/examples/template/template.yaml +++ b/packages/create-app/templates/default-app/examples/template/template.yaml @@ -71,4 +71,4 @@ spec: url: ${{ steps["publish"].output.remoteUrl }} - title: Open in catalog icon: catalog - entityRef: ${{ steps["register"].output.entityRef }} + entityRef: ${{ steps['register'].output.entityRef }} diff --git a/plugins/scaffolder-backend-module-cookiecutter/README.md b/plugins/scaffolder-backend-module-cookiecutter/README.md index 642ef25edf..8516139e64 100644 --- a/plugins/scaffolder-backend-module-cookiecutter/README.md +++ b/plugins/scaffolder-backend-module-cookiecutter/README.md @@ -129,7 +129,7 @@ spec: name: Register action: catalog:register input: - repoContentsUrl: ${{ steps.publish.output.repoContentsUrl }} + repoContentsUrl: ${{ steps['publish'].output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' - name: Results @@ -141,10 +141,10 @@ spec: output: links: - title: Repository - url: ${{ steps.publish.output.remoteUrl }} + url: ${{ steps['publish'].output.remoteUrl }} - title: Open in catalog icon: catalog - entityRef: ${{ steps.register.output.entityRef }} + entityRef: ${{ steps['register'].output.entityRef }} ``` You can also visit the `/create/actions` route in your Backstage application to find out more about the parameters this action accepts when it's installed to configure how you like. diff --git a/plugins/scaffolder-backend-module-rails/README.md b/plugins/scaffolder-backend-module-rails/README.md index 9d1316473d..738cd43cff 100644 --- a/plugins/scaffolder-backend-module-rails/README.md +++ b/plugins/scaffolder-backend-module-rails/README.md @@ -206,7 +206,7 @@ spec: name: Register action: catalog:register input: - repoContentsUrl: ${{ steps.publish.output.repoContentsUrl }} + repoContentsUrl: ${{ steps['publish'].output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' - name: Results @@ -218,10 +218,10 @@ spec: output: links: - title: Repository - url: ${{ steps.publish.output.remoteUrl }} + url: ${{ steps['publish'].output.remoteUrl }} - title: Open in catalog icon: catalog - entityRef: ${{ steps.register.output.entityRef }} + entityRef: ${{ steps['register'].output.entityRef }} ``` ### What you need to run that action diff --git a/plugins/scaffolder-backend-module-yeoman/README.md b/plugins/scaffolder-backend-module-yeoman/README.md index 8ad3a74fe2..5242ef298c 100644 --- a/plugins/scaffolder-backend-module-yeoman/README.md +++ b/plugins/scaffolder-backend-module-yeoman/README.md @@ -126,7 +126,7 @@ spec: name: Register action: catalog:register input: - repoContentsUrl: ${{ steps.publish.output.repoContentsUrl }} + repoContentsUrl: ${{ steps['publish'].output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' - name: Results @@ -138,10 +138,10 @@ spec: output: links: - title: Repository - url: ${{ steps.publish.output.remoteUrl }} + url: ${{ steps['publish'].output.remoteUrl }} - title: Open in catalog icon: catalog - entityRef: ${{ steps.register.output.entityRef }} + entityRef: ${{ steps['register'].output.entityRef }} ``` You can also visit the `/create/actions` route in your Backstage application to find out more about the parameters this action accepts when it's installed to configure how you like. diff --git a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml index 8fd60b594a..074f74f49d 100644 --- a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml @@ -74,7 +74,7 @@ spec: output: links: - title: Repository - url: ${{ steps["publish"].output.remoteUrl }} + url: ${{ steps['publish'].output.remoteUrl }} - title: Open in catalog icon: catalog - entityRef: ${{ steps["register"].output.entityRef }} + entityRef: ${{ steps['register'].output.entityRef }} From c0097db1c8168035456177def0da9b94c6c85891 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Wed, 7 Dec 2022 13:21:52 +0000 Subject: [PATCH 082/437] more fixes and changesets Signed-off-by: Brian Fletcher --- .changeset/dirty-ads-refuse.md | 3 +++ .../templates/default-app/examples/template/template.yaml | 4 ++-- .../sample-templates/bitbucket-demo/template.yaml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.changeset/dirty-ads-refuse.md b/.changeset/dirty-ads-refuse.md index 9c982199e7..848800636e 100644 --- a/.changeset/dirty-ads-refuse.md +++ b/.changeset/dirty-ads-refuse.md @@ -1,4 +1,7 @@ --- +'@backstage/plugin-scaffolder-backend-module-cookiecutter': patch +'@backstage/plugin-scaffolder-backend-module-rails': patch +'@backstage/plugin-scaffolder-backend-module-yeoman': patch '@backstage/create-app': patch '@backstage/plugin-scaffolder-backend': patch --- diff --git a/packages/create-app/templates/default-app/examples/template/template.yaml b/packages/create-app/templates/default-app/examples/template/template.yaml index be3533162f..33f262b49c 100644 --- a/packages/create-app/templates/default-app/examples/template/template.yaml +++ b/packages/create-app/templates/default-app/examples/template/template.yaml @@ -61,14 +61,14 @@ spec: name: Register action: catalog:register input: - repoContentsUrl: ${{ steps["publish"].output.repoContentsUrl }} + repoContentsUrl: ${{ steps['publish'].output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' # Outputs are displayed to the user after a successful execution of the template. output: links: - title: Repository - url: ${{ steps["publish"].output.remoteUrl }} + url: ${{ steps['publish'].output.remoteUrl }} - title: Open in catalog icon: catalog entityRef: ${{ steps['register'].output.entityRef }} diff --git a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml index 074f74f49d..ee1bc634b4 100644 --- a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml @@ -68,7 +68,7 @@ spec: name: Register action: catalog:register input: - repoContentsUrl: ${{ steps["publish"].output.repoContentsUrl }} + repoContentsUrl: ${{ steps['publish'].output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' output: From 2f52b1274009c3542f578d290a894885f5d1b75e Mon Sep 17 00:00:00 2001 From: Scott Guymer Date: Wed, 7 Dec 2022 17:13:11 +0100 Subject: [PATCH 083/437] Remove cleanup calls as they are handled in the setup of msw Signed-off-by: Scott Guymer --- .../search/StackOverflowQuestionsCollatorFactory.test.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts index 9f2c316be4..bdc5be125c 100644 --- a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts +++ b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts @@ -95,14 +95,6 @@ describe('StackOverflowQuestionsCollatorFactory', () => { const worker = setupServer(); setupRequestMockHandlers(worker); - afterEach(async () => { - worker.resetHandlers(); - }); - - afterAll(async () => { - worker.close(); - }); - it('returns a readable stream', async () => { const factory = StackOverflowQuestionsCollatorFactory.fromConfig( config, From c773242555dab1008e68644f82e795e25230c9bd Mon Sep 17 00:00:00 2001 From: Sarah Medeiros Date: Wed, 7 Dec 2022 14:08:46 -0500 Subject: [PATCH 084/437] Remove check from useMemo in order to keep function pure Signed-off-by: Sarah Medeiros --- .../EntityLifecyclePicker.tsx | 38 +++++++-------- .../EntityOwnerPicker/EntityOwnerPicker.tsx | 46 +++++++++---------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx index 693c8ab85c..dda24c3639 100644 --- a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx +++ b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx @@ -67,14 +67,6 @@ export const EntityLifecyclePicker = () => { : filters.lifecycles?.values ?? [], ); - // Set selected lifecycles on query parameter updates; this happens at initial page load and from - // external updates to the page location. - useEffect(() => { - if (queryParamLifecycles.length) { - setSelectedLifecycles(queryParamLifecycles); - } - }, [queryParamLifecycles]); - useEffect(() => { updateFilters({ lifecycles: selectedLifecycles.length @@ -83,17 +75,25 @@ export const EntityLifecyclePicker = () => { }); }, [selectedLifecycles, updateFilters]); - const availableLifecycles = useMemo(() => { - const lifecycles = [ - ...new Set( - backendEntities - .map((e: Entity) => e.spec?.lifecycle) - .filter(Boolean) as string[], - ), - ].sort(); - if (lifecycles.length === 0) setSelectedLifecycles([]); - return lifecycles; - }, [backendEntities]); + const availableLifecycles = useMemo( + () => + [ + ...new Set( + backendEntities + .map((e: Entity) => e.spec?.lifecycle) + .filter(Boolean) as string[], + ), + ].sort(), + [backendEntities], + ); + + // Set selected lifecycles on query parameter updates; this happens at initial page load and from + // external updates to the page location. + useEffect(() => { + if (queryParamLifecycles.length && availableLifecycles.length) { + setSelectedLifecycles(queryParamLifecycles); + } + }, [queryParamLifecycles, availableLifecycles]); if (!availableLifecycles.length) return null; diff --git a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx index d389ead06a..3804324619 100644 --- a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx +++ b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx @@ -67,14 +67,6 @@ export const EntityOwnerPicker = () => { queryParamOwners.length ? queryParamOwners : filters.owners?.values ?? [], ); - // Set selected owners on query parameter updates; this happens at initial page load and from - // external updates to the page location. - useEffect(() => { - if (queryParamOwners.length) { - setSelectedOwners(queryParamOwners); - } - }, [queryParamOwners]); - useEffect(() => { updateFilters({ owners: selectedOwners.length @@ -83,21 +75,29 @@ export const EntityOwnerPicker = () => { }); }, [selectedOwners, updateFilters]); - const availableOwners = useMemo(() => { - const owners = [ - ...new Set( - backendEntities - .flatMap((e: Entity) => - getEntityRelations(e, RELATION_OWNED_BY).map(o => - humanizeEntityRef(o, { defaultKind: 'group' }), - ), - ) - .filter(Boolean) as string[], - ), - ].sort(); - if (owners.length === 0) setSelectedOwners([]); - return owners; - }, [backendEntities]); + const availableOwners = useMemo( + () => + [ + ...new Set( + backendEntities + .flatMap((e: Entity) => + getEntityRelations(e, RELATION_OWNED_BY).map(o => + humanizeEntityRef(o, { defaultKind: 'group' }), + ), + ) + .filter(Boolean) as string[], + ), + ].sort(), + [backendEntities], + ); + + // Set selected owners on query parameter updates; this happens at initial page load and from + // external updates to the page location. + useEffect(() => { + if (queryParamOwners.length && availableOwners.length) { + setSelectedOwners(queryParamOwners); + } + }, [queryParamOwners, availableOwners]); if (!availableOwners.length) return null; From 50aa75ed8042133a5607f84b0b249c6e1d08572c Mon Sep 17 00:00:00 2001 From: Sarah Medeiros Date: Wed, 7 Dec 2022 14:25:01 -0500 Subject: [PATCH 085/437] Add available filters check back Signed-off-by: Sarah Medeiros --- .../EntityLifecyclePicker/EntityLifecyclePicker.tsx | 4 ++++ .../src/components/EntityOwnerPicker/EntityOwnerPicker.tsx | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx index dda24c3639..c19da47feb 100644 --- a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx +++ b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx @@ -95,6 +95,10 @@ export const EntityLifecyclePicker = () => { } }, [queryParamLifecycles, availableLifecycles]); + useEffect(() => { + if (!availableLifecycles.length) setSelectedLifecycles([]); + }, [availableLifecycles]); + if (!availableLifecycles.length) return null; return ( diff --git a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx index 3804324619..894b82c92d 100644 --- a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx +++ b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx @@ -99,6 +99,10 @@ export const EntityOwnerPicker = () => { } }, [queryParamOwners, availableOwners]); + useEffect(() => { + if (!availableOwners.length) setSelectedOwners([]); + }, [availableOwners]); + if (!availableOwners.length) return null; return ( From 449cd409ffa2701375e2874f3debded19546129c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 7 Dec 2022 21:23:54 +0000 Subject: [PATCH 086/437] Update dependency @react-hookz/web to v20.0.1 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 34da101e3c..1d3b3b0225 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12426,8 +12426,8 @@ __metadata: linkType: hard "@react-hookz/web@npm:^20.0.0": - version: 20.0.0 - resolution: "@react-hookz/web@npm:20.0.0" + version: 20.0.1 + resolution: "@react-hookz/web@npm:20.0.1" dependencies: "@react-hookz/deep-equal": ^1.0.3 peerDependencies: @@ -12437,7 +12437,7 @@ __metadata: peerDependenciesMeta: js-cookie: optional: true - checksum: 475d03cdd9a9131b7094549602c8a3ab52ad33e99d7ef408ba2ee7897bfdf332ab5228f46187b9a4c42535b4aaaf808e2c9c882949b2ec32d5ad42d53487a723 + checksum: 433dcb140953a877d4c81e1c176cd4302c184293ccddc5ee8de3ae3e592441b528c0c88bc07ed29fea2c993bacb146dfe0c7541832cb56785f07d8717626a1a2 languageName: node linkType: hard From 846898e56090657392f3909447c1356366583799 Mon Sep 17 00:00:00 2001 From: Tomas Coufal Date: Thu, 8 Dec 2022 08:57:03 +0100 Subject: [PATCH 087/437] chore: run prettier against ADOPTERS.md Signed-off-by: Tomas Coufal --- ADOPTERS.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ADOPTERS.md b/ADOPTERS.md index 90aaa5c69b..53084388db 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -22,10 +22,10 @@ _You can do this by using the [Adopter form](https://info.backstage.spotify.com/ | [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | | [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | | [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | -| [Expedia Group](https://www.expediagroup.com) | [@gman0922](https://github.com/gman0922), [Sheena Sharma](mailto:shesharma@expediagroup.com), [Alekhya Karuturi](mailto:akaruturi@expediagroup.com) | EG Developer Front Door | +| [Expedia Group](https://www.expediagroup.com) | [@gman0922](https://github.com/gman0922), [Sheena Sharma](mailto:shesharma@expediagroup.com), [Alekhya Karuturi](mailto:akaruturi@expediagroup.com) | EG Developer Front Door | | [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | | [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | -| [Lunar](https://lunar.app) | [Bjørn Hald Sørensen](https://github.com/crevil) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | +| [Lunar](https://lunar.app) | [Bjørn Hald Sørensen](https://github.com/crevil) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | | [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | | [Peloton](https://www.onepeloton.com/) | [Matt Waldron](https://github.com/daftgopher) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | | [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a team’s engineering dependencies. | @@ -71,7 +71,7 @@ _You can do this by using the [Adopter form](https://info.backstage.spotify.com/ | [Signal Iduna Group](https://www.signal-iduna.de/) | [Jonas Thomsen](https://github.com/JoThomsen) | Developer Portal, documentation, monitoring, service catalog for our insurance ecosystem | | [Tradeshift](https://www.tradeshift.com/) | [Soren Mathiasen](https://github.com/sorenmat) | Developer Portal: documentation, monitoring, service templates, service catalog for our micro services | | [Unity](https://unity.com) | [Ted Cordery](https://github.com/TeddyBallGame) | A centralized service catalog with documentation for our service engineers. | -| [PicPay](https://www.picpay.com) | [Elton Welsch](https://github.com/eltonwelsch), [Emanuella Okada](https://github.com/ManuOkadaPicPay), [PicPay](https://github.com/picpay) | Developer portal for building services through templates, service catalog with ownership of services, documentation and metrics providing autonomy and visibility for all. | +| [PicPay](https://www.picpay.com) | [Elton Welsch](https://github.com/eltonwelsch), [Emanuella Okada](https://github.com/ManuOkadaPicPay), [PicPay](https://github.com/picpay) | Developer portal for building services through templates, service catalog with ownership of services, documentation and metrics providing autonomy and visibility for all. | | [Epic Games](https://www.epicgames.com) | [Brian Jung](https://github.com/brian-at-epic), [Jeff Goldian](https://github.com/jeffgoldian-Epic) | Developer Portal: Service Catalog, Documentation, Software Templates and more making our internal teams' lives easier! | | [Globo](https://globo.com) | [Carlos Gusmão](https://github.com/caeugusmao), [Guilherme Vierno](https://github.com/vierno), [Denis Aoki](https://github.com/dnsaoki2), [Maycon Dionisio](https://github.com/MayconDionisio), | Reduce the friction of accessing the information engineers need about Globo's digital services through a coherent and centralized experience. | | [QBE](https://www.qbe.com/) | [Daniel Steel](https://github.com/danielsteelqbe), [Pete Jespers](https://github.com/petejespersqbe) | Developer portal allowing our global teams to explore and create applications, documentation and cloud infrastructure easily and quickly 🚀 | @@ -141,7 +141,7 @@ _You can do this by using the [Adopter form](https://info.backstage.spotify.com/ | [Kambi AB](https://www.kambi.com) | [Martin Norum](mailto:martin.norum@kambi.com) | We want to kick ass at speed, so we're currently building up a catalog of our existing software, and looking into how Backstage can support us in our journey towards autonomous product teams. Both to improve speed to market and operational awareness. | | [ANZ](https://www.anz.com.au/personal/) | [Elliot Jackson](mailto:elliot.jackson@anz.com) | Catalog, tech docs and automation | | [Genie Solutions](https://www.geniesolutionssoftware.com.au) | [Zainab Bagasrawala](mailto:zainabbagasrawala@geniesolutions.com.au) | Developer Portal to track our projects, documentation, observability tools and more | -| [MadeiraMadeira](https://www.madeiramadeira.com.br) | [DX Team](mailto:dxteam@madeiramadeira.com.br) | As a support tool for developers, following the principles of "Developer Experience". In order to make the developer's day to day more practical, efficient and, why not, happy. | +| [MadeiraMadeira](https://www.madeiramadeira.com.br) | [DX Team](mailto:dxteam@madeiramadeira.com.br) | As a support tool for developers, following the principles of "Developer Experience". In order to make the developer's day to day more practical, efficient and, why not, happy. | | [Sonatype](https://www.sonatype.com) | [Srikar Ananthula](mailto:sananthula@sonatype.com) | Centralize services used internally with many plugins | | [CVS Health](https://www.cvshealth.com) | [Ari Ben-Elazar](mailto:abenelazar@gmail.com) | Cataloging and documenting our service offerings to offer our internal developers a better operational journey | | [Yatra.com](https://www.yatra.com) | [Matiur Rahman Maitur](mailto:arifrahman4u@gmail.com) | Easy to find out Project details, ownership, dependent services, Documentation, it is very useful for developer. | @@ -215,10 +215,10 @@ _You can do this by using the [Adopter form](https://info.backstage.spotify.com/ | [Ferrovial](https://ferrovial.com) | [Jose Luis Rosado](mailto:jlrosado@ferrovial.com) | Backstage is helping us to improve and acelerate dev experience helping teams to quickly find technical documentation, infrastructure templates, pipelines, software components and quickstarters that have been developed by our squads in a inner source friendly environment. | | [Inter&Co](https://bancointer.com.br) | [Arnaud Lanna](https://github.com/arnaudlanna), [Adriano Silva](https://github.com/adrianovss), [Bruno Grossi](https://github.com/begrossi) | We're using Backstage as our internal Developer Portal to catalog and collect repositories and microservices pieces of information like ownership, deployment time, and documentation. | | [StatusNeo](https://statusneo.com/) | [Karan Nangru](mailto:nangru@statusneo.com), [@NishkarshRaj](https://github.com/NishkarshRaj), and [Gaurav Sarien](mailto:gaurav.sarien@statusneo.com) | Harnessing the power of central catalog inventory and self-serving software templates | -| [Alaska Airlines](https://alaskaair.com) | [@swerdick](https://github.com/swerdick) | Backstage is the developer portal for our 'software delivery platform'. Consolidating developer tools to one place, and providing automation to make it easy for developers to create and deploy applications to Kubernetes -| [Loft](https://loft.com.br) | [Squad DevTools](mailto:squad_devtools@loft.com.br) | We're using Backstage to give visibility and promote ownership of all our applications, resources and tools. Now moving to use it as a Developer Portal to create applications, AWS resources etc. | -| [Raízen](https://www.raizen.com) | [Melquisedque Bernardes Pereira](https://github.com/rayleshh) and [Paulo Eduardo Peixoto](https://github.com/padupe) | Backstage helps us to organize and make available, in a simple and direct way, all the infrastructure for new projects. In addition, it has become a great support tool for our developers. | -| [Trifork](https://trifork.com) | [Casper Thygesen](https://github.com/cthtrifork) | We're using Backstage as part of our dataplatform product. It integrates with the infrastructure components and is the developer portal for all the platform users. | -| [MSCI](http://msci.com) | [Stephen Burrows](mailto:stephen.burrows@msci.com) | Developer portal, service catalog, documentation and tooling -| [ESW](https://esw.com) | [Alisson Fabiano](https://github.com/afabianoo), [Bruno Quintella](https://github.com/quintelab) and [Guilherme Oenning](https://github.com/goenning) | Backstage is our one stop shop to find everything related to all our services, such as ownership, dependencies, production status, tech health and much more. -| [FanDuel](https://fanduel.com) | [Diego Herrera](https://github.com/diegoh), [Christy Campbell](https://github.com/FD-ChristopherCampbell) | We use backstage as our developer portal to provide visibility of our software, ownership, strategy, and the state of maturity across disciplines. +| [Alaska Airlines](https://alaskaair.com) | [@swerdick](https://github.com/swerdick) | Backstage is the developer portal for our 'software delivery platform'. Consolidating developer tools to one place, and providing automation to make it easy for developers to create and deploy applications to Kubernetes | +| [Loft](https://loft.com.br) | [Squad DevTools](mailto:squad_devtools@loft.com.br) | We're using Backstage to give visibility and promote ownership of all our applications, resources and tools. Now moving to use it as a Developer Portal to create applications, AWS resources etc. | +| [Raízen](https://www.raizen.com) | [Melquisedque Bernardes Pereira](https://github.com/rayleshh) and [Paulo Eduardo Peixoto](https://github.com/padupe) | Backstage helps us to organize and make available, in a simple and direct way, all the infrastructure for new projects. In addition, it has become a great support tool for our developers. | +| [Trifork](https://trifork.com) | [Casper Thygesen](https://github.com/cthtrifork) | We're using Backstage as part of our dataplatform product. It integrates with the infrastructure components and is the developer portal for all the platform users. | +| [MSCI](http://msci.com) | [Stephen Burrows](mailto:stephen.burrows@msci.com) | Developer portal, service catalog, documentation and tooling | +| [ESW](https://esw.com) | [Alisson Fabiano](https://github.com/afabianoo), [Bruno Quintella](https://github.com/quintelab) and [Guilherme Oenning](https://github.com/goenning) | Backstage is our one stop shop to find everything related to all our services, such as ownership, dependencies, production status, tech health and much more. | +| [FanDuel](https://fanduel.com) | [Diego Herrera](https://github.com/diegoh), [Christy Campbell](https://github.com/FD-ChristopherCampbell) | We use backstage as our developer portal to provide visibility of our software, ownership, strategy, and the state of maturity across disciplines. | From 8a209b7defb35fe5e4319c56c723b3e930082ab7 Mon Sep 17 00:00:00 2001 From: Tomas Coufal Date: Thu, 8 Dec 2022 08:58:08 +0100 Subject: [PATCH 088/437] docs(addopters): Add Operate First to ADOPTERS.md Signed-off-by: Tomas Coufal --- ADOPTERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ADOPTERS.md b/ADOPTERS.md index 53084388db..7ec300be6a 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -222,3 +222,4 @@ _You can do this by using the [Adopter form](https://info.backstage.spotify.com/ | [MSCI](http://msci.com) | [Stephen Burrows](mailto:stephen.burrows@msci.com) | Developer portal, service catalog, documentation and tooling | | [ESW](https://esw.com) | [Alisson Fabiano](https://github.com/afabianoo), [Bruno Quintella](https://github.com/quintelab) and [Guilherme Oenning](https://github.com/goenning) | Backstage is our one stop shop to find everything related to all our services, such as ownership, dependencies, production status, tech health and much more. | | [FanDuel](https://fanduel.com) | [Diego Herrera](https://github.com/diegoh), [Christy Campbell](https://github.com/FD-ChristopherCampbell) | We use backstage as our developer portal to provide visibility of our software, ownership, strategy, and the state of maturity across disciplines. | +| [Operate First](https://www.operate-first.cloud/) | [Tom Coufal](https://github.com/tumido), [Sam Kopecky](https://github.com/samokopecky) | Backstage provides us with a public service catalog and serves as a gateway to our community cloud. Our instance is publicly available to everyone [here](https://service-catalog.operate-first.cloud/) ([source](https://github.com/operate-first/service-catalog)) | From e7ab201984937d580ad8a7d5467d9c6a83dfe0a1 Mon Sep 17 00:00:00 2001 From: Dmytro Shamenko Date: Wed, 7 Dec 2022 18:43:40 +0200 Subject: [PATCH 089/437] added Affinidi to adopters Signed-off-by: Dmytro Shamenko --- ADOPTERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ADOPTERS.md b/ADOPTERS.md index 90aaa5c69b..f716d0d9ac 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -222,3 +222,4 @@ _You can do this by using the [Adopter form](https://info.backstage.spotify.com/ | [MSCI](http://msci.com) | [Stephen Burrows](mailto:stephen.burrows@msci.com) | Developer portal, service catalog, documentation and tooling | [ESW](https://esw.com) | [Alisson Fabiano](https://github.com/afabianoo), [Bruno Quintella](https://github.com/quintelab) and [Guilherme Oenning](https://github.com/goenning) | Backstage is our one stop shop to find everything related to all our services, such as ownership, dependencies, production status, tech health and much more. | [FanDuel](https://fanduel.com) | [Diego Herrera](https://github.com/diegoh), [Christy Campbell](https://github.com/FD-ChristopherCampbell) | We use backstage as our developer portal to provide visibility of our software, ownership, strategy, and the state of maturity across disciplines. +| [Affinidi](https://affinid.com) | [Dmytro Shamenko](https://github.com/idestis), [Denis Fastovets](https://www.linkedin.com/in/denis-fastovets/) | The Backstage is used in the company to enhance development acceleration inside of the company and helps us keep the house in order, from cost efficiency and up to accountability. From 840f2113c6d889f7edf6eb94dd0866641162db9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 8 Dec 2022 11:23:08 +0100 Subject: [PATCH 090/437] Handle missing commits in GitlabUrlReader.readTree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/mighty-parrots-hammer.md | 5 ++ .../src/reading/GitlabUrlReader.test.ts | 51 ++++++++++++++----- .../src/reading/GitlabUrlReader.ts | 3 +- 3 files changed, 43 insertions(+), 16 deletions(-) create mode 100644 .changeset/mighty-parrots-hammer.md diff --git a/.changeset/mighty-parrots-hammer.md b/.changeset/mighty-parrots-hammer.md new file mode 100644 index 0000000000..5a39d8955e --- /dev/null +++ b/.changeset/mighty-parrots-hammer.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Fix `GitlabUrlReader.readTree` bug when there were no matching commits diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 31092e7707..3751ee868e 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -229,22 +229,28 @@ describe('GitlabUrlReader', () => { path.resolve(__dirname, '__fixtures__/gitlab-archive.tar.gz'), ); - const projectGitlabApiResponse = { - id: 11111111, - default_branch: 'main', - }; + let projectGitlabApiResponse: any; + let commitsGitlabApiResponse: any; + let specificPathCommitsGitlabApiResponse: any; - const commitsGitlabApiResponse = [ - { - id: 'sha123abc', - }, - ]; + beforeEach(() => { + projectGitlabApiResponse = { + id: 11111111, + default_branch: 'main', + }; - const specificPathCommitsGitlabApiResponse = [ - { - id: 'sha456def', - }, - ]; + commitsGitlabApiResponse = [ + { + id: 'sha123abc', + }, + ]; + + specificPathCommitsGitlabApiResponse = [ + { + id: 'sha456def', + }, + ]; + }); beforeEach(() => { worker.use( @@ -494,6 +500,23 @@ describe('GitlabUrlReader', () => { }; await expect(fnGitlab).rejects.toThrow(NotFoundError); }); + + it('should gracefully handle no matching commits', async () => { + commitsGitlabApiResponse = []; + + const response = await gitlabProcessor.readTree( + 'https://gitlab.com/backstage/mock/tree/main', + ); + + const files = await response.files(); + expect(files.length).toBe(2); + + const indexMarkdownFile = await files[0].content(); + const mkDocsFile = await files[1].content(); + + expect(mkDocsFile.toString()).toBe('site_name: Test\n'); + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); }); describe('search', () => { diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index a582f02cf0..c80f1b566c 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -188,8 +188,7 @@ export class GitlabUrlReader implements UrlReader { throw new Error(message); } - const commitSha = (await commitsGitlabResponse.json())[0].id; - + const commitSha = (await commitsGitlabResponse.json())[0]?.id ?? ''; if (etag && etag === commitSha) { throw new NotModifiedError(); } From e0d9c9559a1abf70a871984d9d94805ba6153fc4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 7 Dec 2022 22:51:37 +0100 Subject: [PATCH 091/437] core-app-api: added AppRouter as replacement for app.getRouter() Signed-off-by: Patrik Oldsberg --- .changeset/fuzzy-rivers-search.md | 5 + packages/core-app-api/api-report.md | 3 + packages/core-app-api/src/app/AppManager.tsx | 148 +------------- packages/core-app-api/src/app/AppRouter.tsx | 189 ++++++++++++++++++ .../src/app/InternalAppContext.ts | 27 +++ packages/core-app-api/src/app/index.ts | 1 + packages/core-app-api/src/app/types.ts | 2 + 7 files changed, 233 insertions(+), 142 deletions(-) create mode 100644 .changeset/fuzzy-rivers-search.md create mode 100644 packages/core-app-api/src/app/AppRouter.tsx create mode 100644 packages/core-app-api/src/app/InternalAppContext.ts diff --git a/.changeset/fuzzy-rivers-search.md b/.changeset/fuzzy-rivers-search.md new file mode 100644 index 0000000000..06e2dda6d7 --- /dev/null +++ b/.changeset/fuzzy-rivers-search.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-app-api': minor +--- + +Added a new `AppRouter` component that replaces the same component currently created through `app.getRouter()`. diff --git a/packages/core-app-api/api-report.md b/packages/core-app-api/api-report.md index 70312a54a5..137d6d72ca 100644 --- a/packages/core-app-api/api-report.md +++ b/packages/core-app-api/api-report.md @@ -227,6 +227,9 @@ export type AppRouteBinder = < >, ) => void; +// @public +export function AppRouter({ children }: { children?: ReactNode }): JSX.Element; + // @public export class AppThemeSelector implements AppThemeApi { constructor(themes: AppTheme[]); diff --git a/packages/core-app-api/src/app/AppManager.tsx b/packages/core-app-api/src/app/AppManager.tsx index f4fa2d8442..a5c981a54e 100644 --- a/packages/core-app-api/src/app/AppManager.tsx +++ b/packages/core-app-api/src/app/AppManager.tsx @@ -17,15 +17,10 @@ import { AppConfig, Config } from '@backstage/config'; import React, { ComponentType, - createContext, PropsWithChildren, - ReactElement, - useContext, useMemo, useRef, - useState, } from 'react'; -import { Route, Routes } from 'react-router-dom'; import useAsync from 'react-use/lib/useAsync'; import { ApiProvider, @@ -34,7 +29,6 @@ import { LocalStorageFeatureFlags, } from '../apis'; import { - useApi, AnyApiFactory, ApiHolder, IconComponent, @@ -44,7 +38,6 @@ import { AppThemeApi, ConfigApi, featureFlagsApiRef, - IdentityApi, identityApiRef, BackstagePlugin, } from '@backstage/core-plugin-api'; @@ -61,7 +54,6 @@ import { routingV2Collector, } from '../routing/collectors'; import { RoutingProvider } from '../routing/RoutingProvider'; -import { RouteTracker } from '../routing/RouteTracker'; import { validateRouteParameters, validateRouteBindings, @@ -74,14 +66,14 @@ import { AppContext, AppOptions, BackstageApp, - SignInPageProps, } from './types'; import { AppThemeProvider } from './AppThemeProvider'; import { defaultConfigLoader } from './defaultConfigLoader'; import { ApiRegistry } from '../apis/system/ApiRegistry'; import { resolveRouteBindings } from './resolveRouteBindings'; -import { BackstageRouteObject } from '../routing/types'; import { isReactRouterBeta } from './isReactRouterBeta'; +import { InternalAppContext } from './InternalAppContext'; +import { AppRouter, getBasePath } from './AppRouter'; type CompatiblePlugin = | BackstagePlugin @@ -89,39 +81,6 @@ type CompatiblePlugin = output(): Array<{ type: 'feature-flag'; name: string }>; }); -const InternalAppContext = createContext<{ - routeObjects: BackstageRouteObject[]; -}>({ routeObjects: [] }); - -/** - * Get the app base path from the configured app baseUrl. - * - * The returned path does not have a trailing slash. - */ -function getBasePath(configApi: Config) { - if (!isReactRouterBeta()) { - // When using rr v6 stable the base path is handled through the - // basename prop on the router component instead. - return ''; - } - - return readBasePath(configApi); -} - -/** - * Read the configured base path. - * - * The returned path does not have a trailing slash. - */ -function readBasePath(configApi: ConfigApi) { - let { pathname } = new URL( - configApi.getOptionalString('app.baseUrl') ?? '/', - 'http://sample.dev', // baseUrl can be specified as just a path - ); - pathname = pathname.replace(/\/*$/, ''); - return pathname; -} - function useConfigLoader( configLoader: AppConfigLoader | undefined, components: AppComponents, @@ -413,7 +372,10 @@ export class AppManager implements BackstageApp { basePath={getBasePath(loadedConfig.api)} > {children} @@ -427,104 +389,6 @@ export class AppManager implements BackstageApp { } getRouter(): ComponentType<{}> { - const { Router: RouterComponent, SignInPage: SignInPageComponent } = - this.components; - - // This wraps the sign-in page and waits for sign-in to be completed before rendering the app - const SignInPageWrapper = ({ - component: Component, - children, - }: { - component: ComponentType; - children: ReactElement; - }) => { - const [identityApi, setIdentityApi] = useState(); - const configApi = useApi(configApiRef); - const basePath = getBasePath(configApi); - - if (!identityApi) { - return ; - } - - this.appIdentityProxy.setTarget(identityApi, { - signOutTargetUrl: basePath || '/', - }); - return children; - }; - - const AppRouter = ({ children }: PropsWithChildren<{}>) => { - const configApi = useApi(configApiRef); - const basePath = readBasePath(configApi); - const mountPath = `${basePath}/*`; - const { routeObjects } = useContext(InternalAppContext); - - // If the app hasn't configured a sign-in page, we just continue as guest. - if (!SignInPageComponent) { - this.appIdentityProxy.setTarget( - { - getUserId: () => 'guest', - getIdToken: async () => undefined, - getProfile: () => ({ - email: 'guest@example.com', - displayName: 'Guest', - }), - getProfileInfo: async () => ({ - email: 'guest@example.com', - displayName: 'Guest', - }), - getBackstageIdentity: async () => ({ - type: 'user', - userEntityRef: 'user:default/guest', - ownershipEntityRefs: ['user:default/guest'], - }), - getCredentials: async () => ({}), - signOut: async () => {}, - }, - { signOutTargetUrl: basePath || '/' }, - ); - - if (isReactRouterBeta()) { - return ( - - - - {children}} /> - - - ); - } - - return ( - - - {children} - - ); - } - - if (isReactRouterBeta()) { - return ( - - - - - {children}} /> - - - - ); - } - - return ( - - - - <>{children} - - - ); - }; - return AppRouter; } diff --git a/packages/core-app-api/src/app/AppRouter.tsx b/packages/core-app-api/src/app/AppRouter.tsx new file mode 100644 index 0000000000..c95bdaa7c5 --- /dev/null +++ b/packages/core-app-api/src/app/AppRouter.tsx @@ -0,0 +1,189 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React, { useContext, ReactNode, ComponentType, useState } from 'react'; +import { + ConfigApi, + configApiRef, + IdentityApi, + SignInPageProps, + useApi, + useApp, +} from '@backstage/core-plugin-api'; +import { InternalAppContext } from './InternalAppContext'; +import { isReactRouterBeta } from './isReactRouterBeta'; +import { RouteTracker } from '../routing/RouteTracker'; +import { Route, Routes } from 'react-router-dom'; +import { AppIdentityProxy } from '../apis/implementations/IdentityApi/AppIdentityProxy'; + +/** + * Get the app base path from the configured app baseUrl. + * + * The returned path does not have a trailing slash. + */ +export function getBasePath(configApi: ConfigApi) { + if (!isReactRouterBeta()) { + // When using rr v6 stable the base path is handled through the + // basename prop on the router component instead. + return ''; + } + + return readBasePath(configApi); +} + +/** + * Read the configured base path. + * + * The returned path does not have a trailing slash. + */ +function readBasePath(configApi: ConfigApi) { + let { pathname } = new URL( + configApi.getOptionalString('app.baseUrl') ?? '/', + 'http://sample.dev', // baseUrl can be specified as just a path + ); + pathname = pathname.replace(/\/*$/, ''); + return pathname; +} + +// This wraps the sign-in page and waits for sign-in to be completed before rendering the app +function SignInPageWrapper({ + component: Component, + appIdentityProxy, + children, +}: { + component: ComponentType; + appIdentityProxy: AppIdentityProxy; + children: ReactNode; +}) { + const [identityApi, setIdentityApi] = useState(); + const configApi = useApi(configApiRef); + const basePath = getBasePath(configApi); + + if (!identityApi) { + return ; + } + + appIdentityProxy.setTarget(identityApi, { + signOutTargetUrl: basePath || '/', + }); + return <>{children}; +} + +/** + * Props for the {@link AppRouter} component. + * @public + */ +export interface AppRouterProps { + children?: ReactNode; +} + +/** + * App router and sign-in page wrapper. + * + * @public + * @remarks + * + * The AppRouter provides the routing context and renders the sign-in page. + * Until the user has successfully signed in, this component will render + * the sign-in page. Once the user has signed-in, it will instead render + * the app, while providing routing and route tracking for the app. + * + */ +export function AppRouter({ children }: { children?: ReactNode }) { + const { Router: RouterComponent, SignInPage: SignInPageComponent } = + useApp().getComponents(); + + const configApi = useApi(configApiRef); + const basePath = readBasePath(configApi); + const mountPath = `${basePath}/*`; + const internalAppContext = useContext(InternalAppContext); + if (!internalAppContext) { + throw new Error('AppRouter must be rendered within the AppProvider'); + } + const { routeObjects, appIdentityProxy } = internalAppContext; + + // If the app hasn't configured a sign-in page, we just continue as guest. + if (!SignInPageComponent) { + appIdentityProxy.setTarget( + { + getUserId: () => 'guest', + getIdToken: async () => undefined, + getProfile: () => ({ + email: 'guest@example.com', + displayName: 'Guest', + }), + getProfileInfo: async () => ({ + email: 'guest@example.com', + displayName: 'Guest', + }), + getBackstageIdentity: async () => ({ + type: 'user', + userEntityRef: 'user:default/guest', + ownershipEntityRefs: ['user:default/guest'], + }), + getCredentials: async () => ({}), + signOut: async () => {}, + }, + { signOutTargetUrl: basePath || '/' }, + ); + + if (isReactRouterBeta()) { + return ( + + + + {children}} /> + + + ); + } + + return ( + + + {children} + + ); + } + + if (isReactRouterBeta()) { + return ( + + + + + {children}} /> + + + + ); + } + + return ( + + + + {children} + + + ); +} diff --git a/packages/core-app-api/src/app/InternalAppContext.ts b/packages/core-app-api/src/app/InternalAppContext.ts new file mode 100644 index 0000000000..81382acf62 --- /dev/null +++ b/packages/core-app-api/src/app/InternalAppContext.ts @@ -0,0 +1,27 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createContext } from 'react'; +import { AppIdentityProxy } from '../apis/implementations/IdentityApi/AppIdentityProxy'; +import { BackstageRouteObject } from '../routing/types'; + +export const InternalAppContext = createContext< + | undefined + | { + routeObjects: BackstageRouteObject[]; + appIdentityProxy: AppIdentityProxy; + } +>(undefined); diff --git a/packages/core-app-api/src/app/index.ts b/packages/core-app-api/src/app/index.ts index 7843b36339..f6289a5830 100644 --- a/packages/core-app-api/src/app/index.ts +++ b/packages/core-app-api/src/app/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +export { AppRouter } from './AppRouter'; export { createSpecializedApp } from './createSpecializedApp'; export { defaultConfigLoader } from './defaultConfigLoader'; export * from './types'; diff --git a/packages/core-app-api/src/app/types.ts b/packages/core-app-api/src/app/types.ts index 70022534a1..861766f8d3 100644 --- a/packages/core-app-api/src/app/types.ts +++ b/packages/core-app-api/src/app/types.ts @@ -307,6 +307,8 @@ export type BackstageApp = { /** * Router component that should wrap the App Routes create with getRoutes() * and any other components that should only be available while signed in. + * + * @deprecated Import and use the {@link AppRouter} component from `@backstage/core-app-api` instead */ getRouter(): ComponentType<{}>; }; From d9b3753f877b5dfb15f8533224dacbbbaa96b545 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 7 Dec 2022 22:56:07 +0100 Subject: [PATCH 092/437] app,create-app: update to use AppRouter Signed-off-by: Patrik Oldsberg --- .changeset/shy-birds-hammer.md | 17 +++++++++++++++++ packages/app/src/App.tsx | 3 +-- .../default-app/packages/app/src/App.tsx | 3 +-- 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 .changeset/shy-birds-hammer.md diff --git a/.changeset/shy-birds-hammer.md b/.changeset/shy-birds-hammer.md new file mode 100644 index 0000000000..132b137407 --- /dev/null +++ b/.changeset/shy-birds-hammer.md @@ -0,0 +1,17 @@ +--- +'@backstage/create-app': patch +--- + +Updated the app template to use the new `AppRouter` component instead of `app.getRouter()`. + +To apply this change to an existing app, make the following change to `packages/app/src/App.tsx`: + +```diff +-import { FlatRoutes } from '@backstage/core-app-api'; ++import { AppRouter, FlatRoutes } from '@backstage/core-app-api'; + + ... + + const AppProvider = app.getProvider(); +-const AppRouter = app.getRouter(); +``` diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index 05ef61c11a..f21872a742 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -27,7 +27,7 @@ import { RELATION_PROVIDES_API, } from '@backstage/catalog-model'; import { createApp } from '@backstage/app-defaults'; -import { FlatRoutes } from '@backstage/core-app-api'; +import { AppRouter, FlatRoutes } from '@backstage/core-app-api'; import { AlertDisplay, OAuthRequestDialog, @@ -146,7 +146,6 @@ const app = createApp({ }); const AppProvider = app.getProvider(); -const AppRouter = app.getRouter(); const routes = ( diff --git a/packages/create-app/templates/default-app/packages/app/src/App.tsx b/packages/create-app/templates/default-app/packages/app/src/App.tsx index 46cb786399..368ed4d679 100644 --- a/packages/create-app/templates/default-app/packages/app/src/App.tsx +++ b/packages/create-app/templates/default-app/packages/app/src/App.tsx @@ -29,7 +29,7 @@ import { Root } from './components/Root'; import { AlertDisplay, OAuthRequestDialog } from '@backstage/core-components'; import { createApp } from '@backstage/app-defaults'; -import { FlatRoutes } from '@backstage/core-app-api'; +import { AppRouter, FlatRoutes } from '@backstage/core-app-api'; import { CatalogGraphPage } from '@backstage/plugin-catalog-graph'; import { RequirePermission } from '@backstage/plugin-permission-react'; import { catalogEntityCreatePermission } from '@backstage/plugin-catalog-common/alpha'; @@ -54,7 +54,6 @@ const app = createApp({ }); const AppProvider = app.getProvider(); -const AppRouter = app.getRouter(); const routes = ( From 06d65c51b25829565c2d0899c4dac92139fcba71 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 7 Dec 2022 23:31:34 +0100 Subject: [PATCH 093/437] core-app-api: add AppRouter tests Signed-off-by: Patrik Oldsberg --- .../core-app-api/src/app/AppRouter.test.tsx | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 packages/core-app-api/src/app/AppRouter.test.tsx diff --git a/packages/core-app-api/src/app/AppRouter.test.tsx b/packages/core-app-api/src/app/AppRouter.test.tsx new file mode 100644 index 0000000000..75f28a6bfe --- /dev/null +++ b/packages/core-app-api/src/app/AppRouter.test.tsx @@ -0,0 +1,120 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { + AppComponents, + configApiRef, + IdentityApi, + identityApiRef, + SignInPageProps, + useApi, +} from '@backstage/core-plugin-api'; +import { InternalAppContext } from './InternalAppContext'; +import { MemoryRouter } from 'react-router-dom'; +import { AppIdentityProxy } from '../apis/implementations/IdentityApi/AppIdentityProxy'; +import { render, screen } from '@testing-library/react'; +import { AppRouter } from './AppRouter'; +import useAsync from 'react-use/lib/useAsync'; +import { AppContextProvider } from './AppContext'; +import { TestApiProvider } from '@backstage/test-utils'; +import { ConfigReader } from '@backstage/config'; + +function UserRefDisplay() { + const identityApi = useApi(identityApiRef); + const { value } = useAsync(() => identityApi.getBackstageIdentity()); + return
ref: {value?.userEntityRef}
; +} + +describe('AppRouter', () => { + const mockComponents = { + Router: MemoryRouter, + } as AppComponents; + + it('should fall back to guest if there is no sign-in page', async () => { + const appIdentityProxy = new AppIdentityProxy(); + + render( + + + mockComponents } as any} + > + + + + + + , + , + ); + + await expect( + screen.findByText('ref: user:default/guest'), + ).resolves.toBeInTheDocument(); + }); + + it('should use the result from the sign-in page', async () => { + const appIdentityProxy = new AppIdentityProxy(); + + const SignInPage = (props: SignInPageProps) => { + props.onSignInSuccess({ + getBackstageIdentity: async () => ({ + type: 'user', + userEntityRef: 'user:default/test', + ownershipEntityRefs: ['user:default/test'], + }), + } as IdentityApi); + return null; + }; + + render( + + + ({ ...mockComponents, SignInPage }), + } as any + } + > + + + + + + , + ); + + await expect( + screen.findByText('ref: user:default/test'), + ).resolves.toBeInTheDocument(); + }); +}); From 62e58de887170999728e028d700bb6ca0cf9e255 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 8 Dec 2022 00:04:15 +0100 Subject: [PATCH 094/437] core-app-api: added app.createRoot() Signed-off-by: Patrik Oldsberg --- .changeset/fuzzy-rivers-search.md | 47 +++++++++++++++++++- packages/core-app-api/api-report.md | 1 + packages/core-app-api/src/app/AppManager.tsx | 16 +++++++ packages/core-app-api/src/app/types.ts | 27 +++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) diff --git a/.changeset/fuzzy-rivers-search.md b/.changeset/fuzzy-rivers-search.md index 06e2dda6d7..4a82d1884c 100644 --- a/.changeset/fuzzy-rivers-search.md +++ b/.changeset/fuzzy-rivers-search.md @@ -2,4 +2,49 @@ '@backstage/core-app-api': minor --- -Added a new `AppRouter` component that replaces the same component currently created through `app.getRouter()`. +Added a new `AppRouter` component and `app.createRoot()` method that replaces `app.getRouter()` and `app.getProvider()`, which are now deprecated. The new `AppRouter` component is a drop-in replacement for the old router component, while the new `app.createRoot()` method is used instead of the old provider component. + +An old app setup might look like this: + +```tsx +const app = createApp(/* ... */); + +const AppProvider = app.getProvider(); +const AppRouter = app.getRouter(); + +const routes = ...; + +const App = () => ( + + + + + {routes} + + +); + +export default App; +``` + +With these new APIs, the setup now looks like this: + +```tsx +import { AppRouter } from '@backstage/core-app-api'; + +const app = createApp(/* ... */); + +const routes = ...; + +export default app.createRoot( + <> + + + + {routes} + + , +); +``` + +Note that `app.createRoot()` accepts a React element, rather than a component. diff --git a/packages/core-app-api/api-report.md b/packages/core-app-api/api-report.md index 137d6d72ca..c07ea55b67 100644 --- a/packages/core-app-api/api-report.md +++ b/packages/core-app-api/api-report.md @@ -262,6 +262,7 @@ export type AuthApiCreateOptions = { export type BackstageApp = { getPlugins(): BackstagePlugin[]; getSystemIcon(key: string): IconComponent | undefined; + createRoot(element: JSX.Element): ComponentType<{}>; getProvider(): ComponentType<{}>; getRouter(): ComponentType<{}>; }; diff --git a/packages/core-app-api/src/app/AppManager.tsx b/packages/core-app-api/src/app/AppManager.tsx index a5c981a54e..5a752ad5df 100644 --- a/packages/core-app-api/src/app/AppManager.tsx +++ b/packages/core-app-api/src/app/AppManager.tsx @@ -248,7 +248,23 @@ export class AppManager implements BackstageApp { return this.components; } + createRoot(element: JSX.Element): ComponentType<{}> { + const AppProvider = this.getProvider(); + const AppRoot = () => { + return {element}; + }; + return AppRoot; + } + + #getProviderCalled = false; getProvider(): ComponentType<{}> { + if (this.#getProviderCalled) { + throw new Error( + 'app.getProvider() or app.createRoot() has already been called, and can only be called once', + ); + } + this.#getProviderCalled = true; + const appContext = new AppContextImpl(this); // We only validate routes once diff --git a/packages/core-app-api/src/app/types.ts b/packages/core-app-api/src/app/types.ts index 861766f8d3..82c05ce268 100644 --- a/packages/core-app-api/src/app/types.ts +++ b/packages/core-app-api/src/app/types.ts @@ -298,9 +298,36 @@ export type BackstageApp = { */ getSystemIcon(key: string): IconComponent | undefined; + /** + * Creates the root component that renders the entire app. + * + * @remarks + * + * This method must only be called once, and you have to provide it the entire + * app element tree. The element tree will be analyzed to discover plugins, + * routes, and other app features. The returned component will render all + * of the app elements wrapped within the app context provider. + * + * @example + * ```tsx + * export default app.createRoot( + * <> + * + * + * + * {routes} + * + * , + * ); + * ``` + */ + createRoot(element: JSX.Element): ComponentType<{}>; + /** * Provider component that should wrap the Router created with getRouter() * and any other components that need to be within the app context. + * + * @deprecated Use {@link BackstageApp.createRoot} instead. */ getProvider(): ComponentType<{}>; From 0e91c1196d1751540f65a0e744f5d3c8b86db235 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 8 Dec 2022 11:43:23 +0100 Subject: [PATCH 095/437] app,create-app: update to use app.createRoot() Signed-off-by: Patrik Oldsberg --- .changeset/shy-birds-hammer.md | 35 +++++++++++++++++-- packages/app/src/App.tsx | 10 ++---- .../default-app/packages/app/src/App.tsx | 10 ++---- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/.changeset/shy-birds-hammer.md b/.changeset/shy-birds-hammer.md index 132b137407..116652f740 100644 --- a/.changeset/shy-birds-hammer.md +++ b/.changeset/shy-birds-hammer.md @@ -2,7 +2,7 @@ '@backstage/create-app': patch --- -Updated the app template to use the new `AppRouter` component instead of `app.getRouter()`. +Updated the app template to use the new `AppRouter` component instead of `app.getRouter()`, as well as `app.createRoot()` instead of `app.getProvider()`. To apply this change to an existing app, make the following change to `packages/app/src/App.tsx`: @@ -12,6 +12,37 @@ To apply this change to an existing app, make the following change to `packages/ ... - const AppProvider = app.getProvider(); +-const AppProvider = app.getProvider(); -const AppRouter = app.getRouter(); + + ... + +-const App = () => ( ++export default app.createRoot( +- ++ <> + + + + {routes} + +- ++ , + ); ``` + +The final export step should end up looking something like this: + +```tsx +export default app.createRoot( + <> + + + + {routes} + + , +); +``` + +Note that `app.createRoot()` accepts a React element, rather than a component. diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index f21872a742..f4a50a0902 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -145,8 +145,6 @@ const app = createApp({ }, }); -const AppProvider = app.getProvider(); - const routes = ( } /> @@ -277,14 +275,12 @@ const routes = ( ); -const App = () => ( - +export default app.createRoot( + <> {routes} - + , ); - -export default App; diff --git a/packages/create-app/templates/default-app/packages/app/src/App.tsx b/packages/create-app/templates/default-app/packages/app/src/App.tsx index 368ed4d679..95fc94703e 100644 --- a/packages/create-app/templates/default-app/packages/app/src/App.tsx +++ b/packages/create-app/templates/default-app/packages/app/src/App.tsx @@ -53,8 +53,6 @@ const app = createApp({ }, }); -const AppProvider = app.getProvider(); - const routes = ( } /> @@ -96,14 +94,12 @@ const routes = ( ); -const App = () => ( - +export default app.createRoot( + <> {routes} - + , ); - -export default App; From dfbdae092eb7c069fb3a4211c2b43a3c8a258e67 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 8 Dec 2022 14:10:13 +0100 Subject: [PATCH 096/437] add techdocs-backend changeset Signed-off-by: Johan Haals --- .changeset/smooth-bulldogs-fix.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/smooth-bulldogs-fix.md diff --git a/.changeset/smooth-bulldogs-fix.md b/.changeset/smooth-bulldogs-fix.md new file mode 100644 index 0000000000..bbbb02aa13 --- /dev/null +++ b/.changeset/smooth-bulldogs-fix.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs-backend': minor +--- + +Added a new optional `accountId` to the configuration options of the AWS S3 publisher. Configuring this option will source credentials for the `accountId` in the `aws` app config section. See https://github.com/backstage/backstage/blob/master/packages/integration-aws-node/README.md for more details. From 46cf6b5dacf248633e9a77244837c876010a357c Mon Sep 17 00:00:00 2001 From: Dmytro Shamenko Date: Thu, 8 Dec 2022 16:11:46 +0200 Subject: [PATCH 097/437] fix typo in affinidi url Signed-off-by: Dmytro Shamenko --- ADOPTERS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ADOPTERS.md b/ADOPTERS.md index f716d0d9ac..e97dd23378 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -222,4 +222,5 @@ _You can do this by using the [Adopter form](https://info.backstage.spotify.com/ | [MSCI](http://msci.com) | [Stephen Burrows](mailto:stephen.burrows@msci.com) | Developer portal, service catalog, documentation and tooling | [ESW](https://esw.com) | [Alisson Fabiano](https://github.com/afabianoo), [Bruno Quintella](https://github.com/quintelab) and [Guilherme Oenning](https://github.com/goenning) | Backstage is our one stop shop to find everything related to all our services, such as ownership, dependencies, production status, tech health and much more. | [FanDuel](https://fanduel.com) | [Diego Herrera](https://github.com/diegoh), [Christy Campbell](https://github.com/FD-ChristopherCampbell) | We use backstage as our developer portal to provide visibility of our software, ownership, strategy, and the state of maturity across disciplines. -| [Affinidi](https://affinid.com) | [Dmytro Shamenko](https://github.com/idestis), [Denis Fastovets](https://www.linkedin.com/in/denis-fastovets/) | The Backstage is used in the company to enhance development acceleration inside of the company and helps us keep the house in order, from cost efficiency and up to accountability. +| [Affinidi](https://affinidi.com) | [Dmytro Shamenko](https://github.com/idestis), [Denis Fastovets](https://www.linkedin.com/in/denis-fastovets/) | The Backstage is used +in the company to enhance development acceleration inside of the company and helps us keep the house in order, from cost efficiency and up to accountability. From 3e3edaea2701199b985ee79032ba54844c76efe0 Mon Sep 17 00:00:00 2001 From: Sarah Medeiros Date: Thu, 8 Dec 2022 10:15:28 -0500 Subject: [PATCH 098/437] Simplify fix Signed-off-by: Sarah Medeiros --- .../EntityLifecyclePicker.tsx | 29 +++++++++---------- .../EntityOwnerPicker/EntityOwnerPicker.tsx | 29 +++++++++---------- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx index c19da47feb..b941f94531 100644 --- a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx +++ b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx @@ -67,13 +67,13 @@ export const EntityLifecyclePicker = () => { : filters.lifecycles?.values ?? [], ); + // Set selected lifecycles on query parameter updates; this happens at initial page load and from + // external updates to the page location. useEffect(() => { - updateFilters({ - lifecycles: selectedLifecycles.length - ? new EntityLifecycleFilter(selectedLifecycles) - : undefined, - }); - }, [selectedLifecycles, updateFilters]); + if (queryParamLifecycles.length) { + setSelectedLifecycles(queryParamLifecycles); + } + }, [queryParamLifecycles]); const availableLifecycles = useMemo( () => @@ -87,17 +87,14 @@ export const EntityLifecyclePicker = () => { [backendEntities], ); - // Set selected lifecycles on query parameter updates; this happens at initial page load and from - // external updates to the page location. useEffect(() => { - if (queryParamLifecycles.length && availableLifecycles.length) { - setSelectedLifecycles(queryParamLifecycles); - } - }, [queryParamLifecycles, availableLifecycles]); - - useEffect(() => { - if (!availableLifecycles.length) setSelectedLifecycles([]); - }, [availableLifecycles]); + updateFilters({ + lifecycles: + selectedLifecycles.length && availableLifecycles.length + ? new EntityLifecycleFilter(selectedLifecycles) + : undefined, + }); + }, [selectedLifecycles, updateFilters, availableLifecycles]); if (!availableLifecycles.length) return null; diff --git a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx index 894b82c92d..b067bda0f5 100644 --- a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx +++ b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx @@ -67,13 +67,13 @@ export const EntityOwnerPicker = () => { queryParamOwners.length ? queryParamOwners : filters.owners?.values ?? [], ); + // Set selected owners on query parameter updates; this happens at initial page load and from + // external updates to the page location. useEffect(() => { - updateFilters({ - owners: selectedOwners.length - ? new EntityOwnerFilter(selectedOwners) - : undefined, - }); - }, [selectedOwners, updateFilters]); + if (queryParamOwners.length) { + setSelectedOwners(queryParamOwners); + } + }, [queryParamOwners]); const availableOwners = useMemo( () => @@ -91,17 +91,14 @@ export const EntityOwnerPicker = () => { [backendEntities], ); - // Set selected owners on query parameter updates; this happens at initial page load and from - // external updates to the page location. useEffect(() => { - if (queryParamOwners.length && availableOwners.length) { - setSelectedOwners(queryParamOwners); - } - }, [queryParamOwners, availableOwners]); - - useEffect(() => { - if (!availableOwners.length) setSelectedOwners([]); - }, [availableOwners]); + updateFilters({ + owners: + selectedOwners.length && availableOwners.length + ? new EntityOwnerFilter(selectedOwners) + : undefined, + }); + }, [selectedOwners, updateFilters, availableOwners]); if (!availableOwners.length) return null; From 2e7a08394d33ff877a84bf1ed3f4b3ea6ed70e8e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 8 Dec 2022 17:18:25 +0100 Subject: [PATCH 099/437] core-app-api: actually use AppRouterProps Signed-off-by: Patrik Oldsberg --- packages/core-app-api/api-report.md | 8 +++++++- packages/core-app-api/src/app/AppRouter.tsx | 11 +++++------ packages/core-app-api/src/app/index.ts | 1 + 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/core-app-api/api-report.md b/packages/core-app-api/api-report.md index c07ea55b67..4d867f0b37 100644 --- a/packages/core-app-api/api-report.md +++ b/packages/core-app-api/api-report.md @@ -228,7 +228,13 @@ export type AppRouteBinder = < ) => void; // @public -export function AppRouter({ children }: { children?: ReactNode }): JSX.Element; +export function AppRouter(props: AppRouterProps): JSX.Element; + +// @public +export interface AppRouterProps { + // (undocumented) + children?: ReactNode; +} // @public export class AppThemeSelector implements AppThemeApi { diff --git a/packages/core-app-api/src/app/AppRouter.tsx b/packages/core-app-api/src/app/AppRouter.tsx index c95bdaa7c5..b799983a3b 100644 --- a/packages/core-app-api/src/app/AppRouter.tsx +++ b/packages/core-app-api/src/app/AppRouter.tsx @@ -100,9 +100,8 @@ export interface AppRouterProps { * Until the user has successfully signed in, this component will render * the sign-in page. Once the user has signed-in, it will instead render * the app, while providing routing and route tracking for the app. - * */ -export function AppRouter({ children }: { children?: ReactNode }) { +export function AppRouter(props: AppRouterProps) { const { Router: RouterComponent, SignInPage: SignInPageComponent } = useApp().getComponents(); @@ -145,7 +144,7 @@ export function AppRouter({ children }: { children?: ReactNode }) { - {children}} /> + {props.children}} /> ); @@ -154,7 +153,7 @@ export function AppRouter({ children }: { children?: ReactNode }) { return ( - {children} + {props.children} ); } @@ -168,7 +167,7 @@ export function AppRouter({ children }: { children?: ReactNode }) { appIdentityProxy={appIdentityProxy} > - {children}} /> + {props.children}} /> @@ -182,7 +181,7 @@ export function AppRouter({ children }: { children?: ReactNode }) { component={SignInPageComponent} appIdentityProxy={appIdentityProxy} > - {children} + {props.children} ); diff --git a/packages/core-app-api/src/app/index.ts b/packages/core-app-api/src/app/index.ts index f6289a5830..156c59d0c7 100644 --- a/packages/core-app-api/src/app/index.ts +++ b/packages/core-app-api/src/app/index.ts @@ -15,6 +15,7 @@ */ export { AppRouter } from './AppRouter'; +export type { AppRouterProps } from './AppRouter'; export { createSpecializedApp } from './createSpecializedApp'; export { defaultConfigLoader } from './defaultConfigLoader'; export * from './types'; From 83e6f96614c18f988a933f6a03d26bb7df06772c Mon Sep 17 00:00:00 2001 From: Dmytro Shamenko Date: Thu, 8 Dec 2022 19:24:49 +0200 Subject: [PATCH 100/437] fix new line issue Signed-off-by: Dmytro Shamenko --- ADOPTERS.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ADOPTERS.md b/ADOPTERS.md index e97dd23378..d2c5ebac1f 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -222,5 +222,4 @@ _You can do this by using the [Adopter form](https://info.backstage.spotify.com/ | [MSCI](http://msci.com) | [Stephen Burrows](mailto:stephen.burrows@msci.com) | Developer portal, service catalog, documentation and tooling | [ESW](https://esw.com) | [Alisson Fabiano](https://github.com/afabianoo), [Bruno Quintella](https://github.com/quintelab) and [Guilherme Oenning](https://github.com/goenning) | Backstage is our one stop shop to find everything related to all our services, such as ownership, dependencies, production status, tech health and much more. | [FanDuel](https://fanduel.com) | [Diego Herrera](https://github.com/diegoh), [Christy Campbell](https://github.com/FD-ChristopherCampbell) | We use backstage as our developer portal to provide visibility of our software, ownership, strategy, and the state of maturity across disciplines. -| [Affinidi](https://affinidi.com) | [Dmytro Shamenko](https://github.com/idestis), [Denis Fastovets](https://www.linkedin.com/in/denis-fastovets/) | The Backstage is used -in the company to enhance development acceleration inside of the company and helps us keep the house in order, from cost efficiency and up to accountability. +| [Affinidi](https://affinidi.com) | [Dmytro Shamenko](https://github.com/idestis), [Denis Fastovets](https://www.linkedin.com/in/denis-fastovets/) | The Backstage is used in the company to enhance development acceleration inside of the company and helps us keep the house in order, from cost efficiency and up to accountability. From 0226c5c962e4ddddc6207b220efcfbdcaa89b867 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 17:39:48 +0000 Subject: [PATCH 101/437] Update dependency @roadiehq/backstage-plugin-travis-ci to v2.1.1 Signed-off-by: Renovate Bot --- yarn.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/yarn.lock b/yarn.lock index 33e5718702..eea8fd9904 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12618,13 +12618,13 @@ __metadata: linkType: hard "@roadiehq/backstage-plugin-travis-ci@npm:^2.0.5": - version: 2.1.0 - resolution: "@roadiehq/backstage-plugin-travis-ci@npm:2.1.0" + version: 2.1.1 + resolution: "@roadiehq/backstage-plugin-travis-ci@npm:2.1.1" dependencies: - "@backstage/catalog-model": ^1.1.2 - "@backstage/core-components": ^0.11.2 - "@backstage/core-plugin-api": ^1.0.7 - "@backstage/plugin-catalog-react": ^1.2.0 + "@backstage/catalog-model": ^1.1.3 + "@backstage/core-components": ^0.12.0 + "@backstage/core-plugin-api": ^1.1.0 + "@backstage/plugin-catalog-react": ^1.2.1 "@backstage/theme": ^0.2.16 "@material-ui/core": ^4.11.3 "@material-ui/icons": ^4.11.2 @@ -12639,7 +12639,7 @@ __metadata: react-dom: ^16.13.1 || ^17.0.0 react-router: 6.0.0-beta.0 || ^6.3.0 react-router-dom: 6.0.0-beta.0 || ^6.3.0 - checksum: 99bd8d242bd954f9fd3c75910353d6cce6672ba605a819828a336316c0c7fd0d2f36bfdf7a82b702e284b8f636c06f50b93ae53af6afc3b12e8f766e7fe2ce9c + checksum: 23cb8760c6d140052fcdc22007b59349332afd565ee2b65e8bba4a1b5c5d409ef6505f292d1e5c62d3c281898b65eb83397aa790480e9e1becda3452ef6d086c languageName: node linkType: hard From cd4e367a598b88690b531945787c2706eff7de16 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 17:40:56 +0000 Subject: [PATCH 102/437] Update dependency @rollup/plugin-commonjs to v23.0.4 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 33e5718702..6f4b6526fa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12644,8 +12644,8 @@ __metadata: linkType: hard "@rollup/plugin-commonjs@npm:^23.0.0": - version: 23.0.3 - resolution: "@rollup/plugin-commonjs@npm:23.0.3" + version: 23.0.4 + resolution: "@rollup/plugin-commonjs@npm:23.0.4" dependencies: "@rollup/pluginutils": ^5.0.1 commondir: ^1.0.1 @@ -12658,7 +12658,7 @@ __metadata: peerDependenciesMeta: rollup: optional: true - checksum: 925289c1694e871065d741ce210621df894febc267b9ca574dfe24e00a48b436f3eaa6a6f6d00312a78c9b51cc0cad7abf13b97fc455395576007c681d7ec58d + checksum: 32d84de06140d4d050c0b402c6a6d858a6f970c3d6f50ea1fba40495b0cd0f977486513639875058287885cfbddc6f8b1ab5ebbe0d28f0364506b372fd4b0bd6 languageName: node linkType: hard From ad5a786e339abba6915307e1ad43c73a014f7140 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 18:20:35 +0000 Subject: [PATCH 103/437] Update dependency tar to v6.1.13 Signed-off-by: Renovate Bot --- yarn.lock | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/yarn.lock b/yarn.lock index eea8fd9904..4a8bf137c6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -28985,6 +28985,15 @@ __metadata: languageName: node linkType: hard +"minipass@npm:^4.0.0": + version: 4.0.0 + resolution: "minipass@npm:4.0.0" + dependencies: + yallist: ^4.0.0 + checksum: 7a609afbf394abfcf9c48e6c90226f471676c8f2a67f07f6838871afb03215ede431d1433feffe1b855455bcb13ef0eb89162841b9796109d6fed8d89790f381 + languageName: node + linkType: hard + "minizlib@npm:^2.0.0, minizlib@npm:^2.1.1, minizlib@npm:^2.1.2": version: 2.1.2 resolution: "minizlib@npm:2.1.2" @@ -36027,16 +36036,16 @@ __metadata: linkType: hard "tar@npm:^6.0.2, tar@npm:^6.1.0, tar@npm:^6.1.11, tar@npm:^6.1.12, tar@npm:^6.1.2": - version: 6.1.12 - resolution: "tar@npm:6.1.12" + version: 6.1.13 + resolution: "tar@npm:6.1.13" dependencies: chownr: ^2.0.0 fs-minipass: ^2.0.0 - minipass: ^3.0.0 + minipass: ^4.0.0 minizlib: ^2.1.1 mkdirp: ^1.0.3 yallist: ^4.0.0 - checksum: 49d72e4420944e7ede2782d6b0826a6ede6cdab23c7de63470917e7a78166bc4d5b1a96279d3d79a85f1ba5a17cd37c0acbb3cbff19a07447691445b8b051c55 + checksum: 8a278bed123aa9f53549b256a36b719e317c8b96fe86a63406f3c62887f78267cea9b22dc6f7007009738509800d4a4dccc444abd71d762287c90f35b002eb1c languageName: node linkType: hard From 96dd950865d4d1af62b80a75eb62f6a5d38dfa31 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 18:22:06 +0000 Subject: [PATCH 104/437] Update dependency typescript to v4.9.4 Signed-off-by: Renovate Bot --- cypress/yarn.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cypress/yarn.lock b/cypress/yarn.lock index 49c85adb2b..ded043cdaf 100644 --- a/cypress/yarn.lock +++ b/cypress/yarn.lock @@ -1415,22 +1415,22 @@ __metadata: linkType: hard "typescript@npm:^4.1.3": - version: 4.9.3 - resolution: "typescript@npm:4.9.3" + version: 4.9.4 + resolution: "typescript@npm:4.9.4" bin: tsc: bin/tsc tsserver: bin/tsserver - checksum: 17b8f816050b412403e38d48eef0e893deb6be522d6dc7caf105e54a72e34daf6835c447735fd2b28b66784e72bfbf87f627abb4818a8e43d1fa8106396128dc + checksum: e782fb9e0031cb258a80000f6c13530288c6d63f1177ed43f770533fdc15740d271554cdae86701c1dd2c83b082cea808b07e97fd68b38a172a83dbf9e0d0ef9 languageName: node linkType: hard "typescript@patch:typescript@^4.1.3#~builtin": - version: 4.9.3 - resolution: "typescript@patch:typescript@npm%3A4.9.3#~builtin::version=4.9.3&hash=a1c5e5" + version: 4.9.4 + resolution: "typescript@patch:typescript@npm%3A4.9.4#~builtin::version=4.9.4&hash=a1c5e5" bin: tsc: bin/tsc tsserver: bin/tsserver - checksum: ef65c22622d864497d0a0c5db693523329b3284c15fe632e93ad9aa059e8dc38ef3bd767d6f26b1e5ecf9446f49bd0f6c4e5714a2eeaf352805dc002479843d1 + checksum: 37f6e2c3c5e2aa5934b85b0fddbf32eeac8b1bacf3a5b51d01946936d03f5377fe86255d4e5a4ae628fd0cd553386355ad362c57f13b4635064400f3e8e05b9d languageName: node linkType: hard From d91bac0013884992da3435f06ca2e889e07f1e59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 8 Dec 2022 19:29:41 +0100 Subject: [PATCH 105/437] add the beginnings of a migrations test for the catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../src/database/migrations.test.ts | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 plugins/catalog-backend/src/database/migrations.test.ts diff --git a/plugins/catalog-backend/src/database/migrations.test.ts b/plugins/catalog-backend/src/database/migrations.test.ts new file mode 100644 index 0000000000..d939d04ac6 --- /dev/null +++ b/plugins/catalog-backend/src/database/migrations.test.ts @@ -0,0 +1,90 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; +import { TestDatabases } from '@backstage/backend-test-utils'; +import fs from 'fs'; + +const migrationsDir = `${__dirname}/../../migrations`; +const migrationsFiles = fs.readdirSync(migrationsDir).sort(); + +async function migrateOnce(knex: Knex): Promise { + await knex.migrate.up({ directory: migrationsDir }); +} + +async function migrateUntilBefore(knex: Knex, target: string): Promise { + const index = migrationsFiles.indexOf(target); + if (index === -1) { + throw new Error(`Migration ${target} not found`); + } + for (let i = 0; i < index; i++) { + await migrateOnce(knex); + } +} + +describe('migrations', () => { + const databases = TestDatabases.create({ + ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + }); + + it.each(databases.eachSupportedId())( + '20221109192547_search_add_original_value_column.js, %p', + async databaseId => { + const knex = await databases.init(databaseId); + + await migrateUntilBefore( + knex, + '20221109192547_search_add_original_value_column.js', + ); + + await knex + .insert({ + entity_id: 'i', + entity_ref: 'k:ns/n', + unprocessed_entity: '{}', + errors: '[]', + next_update_at: new Date(), + last_discovery_at: new Date(), + }) + .into('refresh_state'); + await knex + .insert({ entity_id: 'i', key: 'k1', value: 'v1' }) + .into('search'); + await knex + .insert({ entity_id: 'i', key: 'k2', value: null }) + .into('search'); + + await expect(knex('search')).resolves.toEqual( + expect.arrayContaining([ + { entity_id: 'i', key: 'k1', value: 'v1' }, + { entity_id: 'i', key: 'k2', value: null }, + ]), + ); + + await knex.migrate.up({ directory: migrationsDir }); + + await expect(knex('search')).resolves.toEqual( + expect.arrayContaining([ + { entity_id: 'i', key: 'k1', value: 'v1', original_value: 'v1' }, + { entity_id: 'i', key: 'k2', value: null, original_value: null }, + ]), + ); + + await knex.destroy(); + }, + 60_000, + ); +}); From 2cb4cd1a7ba7a96cbcebb993c2514059766d46de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 8 Dec 2022 20:04:11 +0100 Subject: [PATCH 106/437] down too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../src/database/migrations.test.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/src/database/migrations.test.ts b/plugins/catalog-backend/src/database/migrations.test.ts index d939d04ac6..8287ed882d 100644 --- a/plugins/catalog-backend/src/database/migrations.test.ts +++ b/plugins/catalog-backend/src/database/migrations.test.ts @@ -21,17 +21,21 @@ import fs from 'fs'; const migrationsDir = `${__dirname}/../../migrations`; const migrationsFiles = fs.readdirSync(migrationsDir).sort(); -async function migrateOnce(knex: Knex): Promise { +async function migrateUpOnce(knex: Knex): Promise { await knex.migrate.up({ directory: migrationsDir }); } +async function migrateDownOnce(knex: Knex): Promise { + await knex.migrate.down({ directory: migrationsDir }); +} + async function migrateUntilBefore(knex: Knex, target: string): Promise { const index = migrationsFiles.indexOf(target); if (index === -1) { throw new Error(`Migration ${target} not found`); } for (let i = 0; i < index; i++) { - await migrateOnce(knex); + await migrateUpOnce(knex); } } @@ -74,7 +78,7 @@ describe('migrations', () => { ]), ); - await knex.migrate.up({ directory: migrationsDir }); + await migrateUpOnce(knex); await expect(knex('search')).resolves.toEqual( expect.arrayContaining([ @@ -83,6 +87,15 @@ describe('migrations', () => { ]), ); + await migrateDownOnce(knex); + + await expect(knex('search')).resolves.toEqual( + expect.arrayContaining([ + { entity_id: 'i', key: 'k1', value: 'v1' }, + { entity_id: 'i', key: 'k2', value: null }, + ]), + ); + await knex.destroy(); }, 60_000, From f5c4aee0c62c04024f86ff3e39bba1c3eda0030b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 19:04:45 +0000 Subject: [PATCH 107/437] Update dependency vm2 to v3.9.13 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 2734d1c742..e9b3f1f7a1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -37667,14 +37667,14 @@ __metadata: linkType: hard "vm2@npm:^3.9.11": - version: 3.9.11 - resolution: "vm2@npm:3.9.11" + version: 3.9.13 + resolution: "vm2@npm:3.9.13" dependencies: acorn: ^8.7.0 acorn-walk: ^8.2.0 bin: vm2: bin/vm2 - checksum: aab39e6e4b59146d24abacd79f490e854a6e058a8b23d93d2be5aca7720778e2605d2cc028ccc4a5f50d3d91b0c38be9a6247a80d2da1a6de09425cc437770b4 + checksum: ee82c130a9d1b45558fac4e95133ae022e69fafd4bec8aada44bb2f0aebe6ff8ae9b1176e7cbafceffec35267e009b518c121ad81a5565da06aa7a50ed3a09bc languageName: node linkType: hard From 9db0246ab7a5f89733362b78ce2e5ea486160747 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 19:17:06 +0000 Subject: [PATCH 108/437] Update react-router monorepo to v6.4.5 Signed-off-by: Renovate Bot --- yarn.lock | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/yarn.lock b/yarn.lock index 2734d1c742..54e4eddd2d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12441,10 +12441,10 @@ __metadata: languageName: node linkType: hard -"@remix-run/router@npm:1.0.4": - version: 1.0.4 - resolution: "@remix-run/router@npm:1.0.4" - checksum: db6b1e111fb6e7e22c3d274bd6cef3054a0a989245894fde5735efd9cb46d18511e528e813ff50bcbdd7eda1c559f4e76e9d0c54396624e33396168d80b09c42 +"@remix-run/router@npm:1.0.5": + version: 1.0.5 + resolution: "@remix-run/router@npm:1.0.5" + checksum: 5d66750b7defc10c80d9748fa003fcfe2af9ef82de3c06be88c926ede36a15b9e971ed2222981db843494e2f77c3de14470efe6dc877530cea774dc31162ec6f languageName: node linkType: hard @@ -32808,15 +32808,15 @@ __metadata: linkType: hard "react-router-dom@npm:^6.3.0": - version: 6.4.4 - resolution: "react-router-dom@npm:6.4.4" + version: 6.4.5 + resolution: "react-router-dom@npm:6.4.5" dependencies: - "@remix-run/router": 1.0.4 - react-router: 6.4.4 + "@remix-run/router": 1.0.5 + react-router: 6.4.5 peerDependencies: react: ">=16.8" react-dom: ">=16.8" - checksum: 95f31ae84979b404db7483309e1063fbf724046103dc8b7be7e6df79f7b81e4e63dcdb587f11d8be38e3aaf62b71f9e0d83344ef8d8c5e962de7819c793de781 + checksum: 09d7841dd52efd2c60947171f95d9d1860cabd7540ac74dede86a6c36a2dd26a645e4928cbc8aa4e1c9d1f906f2ef144ba6da5ac69243590df855a6df8cfe1aa languageName: node linkType: hard @@ -32831,14 +32831,14 @@ __metadata: languageName: node linkType: hard -"react-router@npm:6.4.4, react-router@npm:^6.3.0": - version: 6.4.4 - resolution: "react-router@npm:6.4.4" +"react-router@npm:6.4.5, react-router@npm:^6.3.0": + version: 6.4.5 + resolution: "react-router@npm:6.4.5" dependencies: - "@remix-run/router": 1.0.4 + "@remix-run/router": 1.0.5 peerDependencies: react: ">=16.8" - checksum: d82cc8b8bdf10e02e07d089f9d82987fa161c2f07583c973126f5f8a052c02b3d2339891ddbda1738f877755a407e2c35f0dc1ac52aaa1d2c853774d19753ea7 + checksum: 0d471df39f0487224240f9910c2f2939519f3e9909a7b19f72767aeea2caf1c9fa4e8ea9a51b17c20051ec252db0fcc34a7a0b3068b338dd9d8eb656ffd489c1 languageName: node linkType: hard From 25e38ebb8ca7c3e1e7e111e4ea91d7e8ff013979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 8 Dec 2022 20:06:23 +0100 Subject: [PATCH 109/437] move to root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- plugins/catalog-backend/src/{database => }/migrations.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename plugins/catalog-backend/src/{database => }/migrations.test.ts (98%) diff --git a/plugins/catalog-backend/src/database/migrations.test.ts b/plugins/catalog-backend/src/migrations.test.ts similarity index 98% rename from plugins/catalog-backend/src/database/migrations.test.ts rename to plugins/catalog-backend/src/migrations.test.ts index 8287ed882d..9d9b02fa82 100644 --- a/plugins/catalog-backend/src/database/migrations.test.ts +++ b/plugins/catalog-backend/src/migrations.test.ts @@ -18,7 +18,7 @@ import { Knex } from 'knex'; import { TestDatabases } from '@backstage/backend-test-utils'; import fs from 'fs'; -const migrationsDir = `${__dirname}/../../migrations`; +const migrationsDir = `${__dirname}/../migrations`; const migrationsFiles = fs.readdirSync(migrationsDir).sort(); async function migrateUpOnce(knex: Knex): Promise { From 6b4b127ff6a7cbc1cac5f64b739cedcf4b1d044b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 20:06:01 +0000 Subject: [PATCH 110/437] Update aws-sdk-js-v3 monorepo to v3.226.0 Signed-off-by: Renovate Bot --- yarn.lock | 1212 +++++++++++++++++++++++++++-------------------------- 1 file changed, 607 insertions(+), 605 deletions(-) diff --git a/yarn.lock b/yarn.lock index 58a62b3920..9c77b522c3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -379,13 +379,13 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/abort-controller@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/abort-controller@npm:3.224.0" +"@aws-sdk/abort-controller@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/abort-controller@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 6d272d21e7cc0aa020f44f97c002b13dd2ad164fe96d6c6caa73b96e429101cb867188b50c452462e8f0092e680fe87f3d033beacda1366c632b5cf41f5b7207 + checksum: 44045b60c7697ed76bcbfbe3f7f4bb019f139d2337e77d8ce79d98bca17c1245e2d33934bba04fe8e2d462b729124ba4414a589ed9275c07f2bfefd3a0850184 languageName: node linkType: hard @@ -408,476 +408,476 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/client-cognito-identity@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/client-cognito-identity@npm:3.224.0" +"@aws-sdk/client-cognito-identity@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/client-cognito-identity@npm:3.226.0" dependencies: "@aws-crypto/sha256-browser": 2.0.0 "@aws-crypto/sha256-js": 2.0.0 - "@aws-sdk/client-sts": 3.224.0 - "@aws-sdk/config-resolver": 3.224.0 - "@aws-sdk/credential-provider-node": 3.224.0 - "@aws-sdk/fetch-http-handler": 3.224.0 - "@aws-sdk/hash-node": 3.224.0 - "@aws-sdk/invalid-dependency": 3.224.0 - "@aws-sdk/middleware-content-length": 3.224.0 - "@aws-sdk/middleware-endpoint": 3.224.0 - "@aws-sdk/middleware-host-header": 3.224.0 - "@aws-sdk/middleware-logger": 3.224.0 - "@aws-sdk/middleware-recursion-detection": 3.224.0 - "@aws-sdk/middleware-retry": 3.224.0 - "@aws-sdk/middleware-serde": 3.224.0 - "@aws-sdk/middleware-signing": 3.224.0 - "@aws-sdk/middleware-stack": 3.224.0 - "@aws-sdk/middleware-user-agent": 3.224.0 - "@aws-sdk/node-config-provider": 3.224.0 - "@aws-sdk/node-http-handler": 3.224.0 - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/smithy-client": 3.224.0 - "@aws-sdk/types": 3.224.0 - "@aws-sdk/url-parser": 3.224.0 + "@aws-sdk/client-sts": 3.226.0 + "@aws-sdk/config-resolver": 3.226.0 + "@aws-sdk/credential-provider-node": 3.226.0 + "@aws-sdk/fetch-http-handler": 3.226.0 + "@aws-sdk/hash-node": 3.226.0 + "@aws-sdk/invalid-dependency": 3.226.0 + "@aws-sdk/middleware-content-length": 3.226.0 + "@aws-sdk/middleware-endpoint": 3.226.0 + "@aws-sdk/middleware-host-header": 3.226.0 + "@aws-sdk/middleware-logger": 3.226.0 + "@aws-sdk/middleware-recursion-detection": 3.226.0 + "@aws-sdk/middleware-retry": 3.226.0 + "@aws-sdk/middleware-serde": 3.226.0 + "@aws-sdk/middleware-signing": 3.226.0 + "@aws-sdk/middleware-stack": 3.226.0 + "@aws-sdk/middleware-user-agent": 3.226.0 + "@aws-sdk/node-config-provider": 3.226.0 + "@aws-sdk/node-http-handler": 3.226.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/smithy-client": 3.226.0 + "@aws-sdk/types": 3.226.0 + "@aws-sdk/url-parser": 3.226.0 "@aws-sdk/util-base64": 3.208.0 "@aws-sdk/util-body-length-browser": 3.188.0 "@aws-sdk/util-body-length-node": 3.208.0 - "@aws-sdk/util-defaults-mode-browser": 3.224.0 - "@aws-sdk/util-defaults-mode-node": 3.224.0 - "@aws-sdk/util-endpoints": 3.224.0 - "@aws-sdk/util-user-agent-browser": 3.224.0 - "@aws-sdk/util-user-agent-node": 3.224.0 + "@aws-sdk/util-defaults-mode-browser": 3.226.0 + "@aws-sdk/util-defaults-mode-node": 3.226.0 + "@aws-sdk/util-endpoints": 3.226.0 + "@aws-sdk/util-user-agent-browser": 3.226.0 + "@aws-sdk/util-user-agent-node": 3.226.0 "@aws-sdk/util-utf8-browser": 3.188.0 "@aws-sdk/util-utf8-node": 3.208.0 tslib: ^2.3.1 - checksum: 0534ed2da499ede416e65a4b92e5f33b9e1c5376ad947447f69bf54fd790b02f141eb054265ed40d694f1bab10e0b2c3ddfe7e7ca62f8c3ce5c86f835d8ab4e6 + checksum: a138f29c7172549f176f3138cd09514d460f62182d5615b89035aff1a2e1e228f3f36db42c7d41be3de07bb4f6a15924aaa3f46b886d7bed4764f2ef12d5ec13 languageName: node linkType: hard "@aws-sdk/client-s3@npm:^3.208.0": - version: 3.224.0 - resolution: "@aws-sdk/client-s3@npm:3.224.0" + version: 3.226.0 + resolution: "@aws-sdk/client-s3@npm:3.226.0" dependencies: "@aws-crypto/sha1-browser": 2.0.0 "@aws-crypto/sha256-browser": 2.0.0 "@aws-crypto/sha256-js": 2.0.0 - "@aws-sdk/client-sts": 3.224.0 - "@aws-sdk/config-resolver": 3.224.0 - "@aws-sdk/credential-provider-node": 3.224.0 - "@aws-sdk/eventstream-serde-browser": 3.224.0 - "@aws-sdk/eventstream-serde-config-resolver": 3.224.0 - "@aws-sdk/eventstream-serde-node": 3.224.0 - "@aws-sdk/fetch-http-handler": 3.224.0 - "@aws-sdk/hash-blob-browser": 3.224.0 - "@aws-sdk/hash-node": 3.224.0 - "@aws-sdk/hash-stream-node": 3.224.0 - "@aws-sdk/invalid-dependency": 3.224.0 - "@aws-sdk/md5-js": 3.224.0 - "@aws-sdk/middleware-bucket-endpoint": 3.224.0 - "@aws-sdk/middleware-content-length": 3.224.0 - "@aws-sdk/middleware-endpoint": 3.224.0 - "@aws-sdk/middleware-expect-continue": 3.224.0 - "@aws-sdk/middleware-flexible-checksums": 3.224.0 - "@aws-sdk/middleware-host-header": 3.224.0 - "@aws-sdk/middleware-location-constraint": 3.224.0 - "@aws-sdk/middleware-logger": 3.224.0 - "@aws-sdk/middleware-recursion-detection": 3.224.0 - "@aws-sdk/middleware-retry": 3.224.0 - "@aws-sdk/middleware-sdk-s3": 3.224.0 - "@aws-sdk/middleware-serde": 3.224.0 - "@aws-sdk/middleware-signing": 3.224.0 - "@aws-sdk/middleware-ssec": 3.224.0 - "@aws-sdk/middleware-stack": 3.224.0 - "@aws-sdk/middleware-user-agent": 3.224.0 - "@aws-sdk/node-config-provider": 3.224.0 - "@aws-sdk/node-http-handler": 3.224.0 - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/signature-v4-multi-region": 3.224.0 - "@aws-sdk/smithy-client": 3.224.0 - "@aws-sdk/types": 3.224.0 - "@aws-sdk/url-parser": 3.224.0 + "@aws-sdk/client-sts": 3.226.0 + "@aws-sdk/config-resolver": 3.226.0 + "@aws-sdk/credential-provider-node": 3.226.0 + "@aws-sdk/eventstream-serde-browser": 3.226.0 + "@aws-sdk/eventstream-serde-config-resolver": 3.226.0 + "@aws-sdk/eventstream-serde-node": 3.226.0 + "@aws-sdk/fetch-http-handler": 3.226.0 + "@aws-sdk/hash-blob-browser": 3.226.0 + "@aws-sdk/hash-node": 3.226.0 + "@aws-sdk/hash-stream-node": 3.226.0 + "@aws-sdk/invalid-dependency": 3.226.0 + "@aws-sdk/md5-js": 3.226.0 + "@aws-sdk/middleware-bucket-endpoint": 3.226.0 + "@aws-sdk/middleware-content-length": 3.226.0 + "@aws-sdk/middleware-endpoint": 3.226.0 + "@aws-sdk/middleware-expect-continue": 3.226.0 + "@aws-sdk/middleware-flexible-checksums": 3.226.0 + "@aws-sdk/middleware-host-header": 3.226.0 + "@aws-sdk/middleware-location-constraint": 3.226.0 + "@aws-sdk/middleware-logger": 3.226.0 + "@aws-sdk/middleware-recursion-detection": 3.226.0 + "@aws-sdk/middleware-retry": 3.226.0 + "@aws-sdk/middleware-sdk-s3": 3.226.0 + "@aws-sdk/middleware-serde": 3.226.0 + "@aws-sdk/middleware-signing": 3.226.0 + "@aws-sdk/middleware-ssec": 3.226.0 + "@aws-sdk/middleware-stack": 3.226.0 + "@aws-sdk/middleware-user-agent": 3.226.0 + "@aws-sdk/node-config-provider": 3.226.0 + "@aws-sdk/node-http-handler": 3.226.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/signature-v4-multi-region": 3.226.0 + "@aws-sdk/smithy-client": 3.226.0 + "@aws-sdk/types": 3.226.0 + "@aws-sdk/url-parser": 3.226.0 "@aws-sdk/util-base64": 3.208.0 "@aws-sdk/util-body-length-browser": 3.188.0 "@aws-sdk/util-body-length-node": 3.208.0 - "@aws-sdk/util-defaults-mode-browser": 3.224.0 - "@aws-sdk/util-defaults-mode-node": 3.224.0 - "@aws-sdk/util-endpoints": 3.224.0 - "@aws-sdk/util-stream-browser": 3.224.0 - "@aws-sdk/util-stream-node": 3.224.0 - "@aws-sdk/util-user-agent-browser": 3.224.0 - "@aws-sdk/util-user-agent-node": 3.224.0 + "@aws-sdk/util-defaults-mode-browser": 3.226.0 + "@aws-sdk/util-defaults-mode-node": 3.226.0 + "@aws-sdk/util-endpoints": 3.226.0 + "@aws-sdk/util-stream-browser": 3.226.0 + "@aws-sdk/util-stream-node": 3.226.0 + "@aws-sdk/util-user-agent-browser": 3.226.0 + "@aws-sdk/util-user-agent-node": 3.226.0 "@aws-sdk/util-utf8-browser": 3.188.0 "@aws-sdk/util-utf8-node": 3.208.0 - "@aws-sdk/util-waiter": 3.224.0 + "@aws-sdk/util-waiter": 3.226.0 "@aws-sdk/xml-builder": 3.201.0 fast-xml-parser: 4.0.11 tslib: ^2.3.1 - checksum: 975e4b8d237b1169a429960d5bfe911339980f5722b858ccc5d9f168abccaeac953a5d68ea21e55ba48e4e8000154280da5170bb6d252adf5847df84b3b1bcf1 + checksum: 5616e74c836dc9f202b5cd73ceb90c580a8a3319cc59a1bfa1dae49f001ade1e38e2a1fc842319ff443470fb63eb000e6acef727f7f95a1b96d360647386900b languageName: node linkType: hard "@aws-sdk/client-sqs@npm:^3.208.0": - version: 3.224.0 - resolution: "@aws-sdk/client-sqs@npm:3.224.0" + version: 3.226.0 + resolution: "@aws-sdk/client-sqs@npm:3.226.0" dependencies: "@aws-crypto/sha256-browser": 2.0.0 "@aws-crypto/sha256-js": 2.0.0 - "@aws-sdk/client-sts": 3.224.0 - "@aws-sdk/config-resolver": 3.224.0 - "@aws-sdk/credential-provider-node": 3.224.0 - "@aws-sdk/fetch-http-handler": 3.224.0 - "@aws-sdk/hash-node": 3.224.0 - "@aws-sdk/invalid-dependency": 3.224.0 - "@aws-sdk/md5-js": 3.224.0 - "@aws-sdk/middleware-content-length": 3.224.0 - "@aws-sdk/middleware-endpoint": 3.224.0 - "@aws-sdk/middleware-host-header": 3.224.0 - "@aws-sdk/middleware-logger": 3.224.0 - "@aws-sdk/middleware-recursion-detection": 3.224.0 - "@aws-sdk/middleware-retry": 3.224.0 - "@aws-sdk/middleware-sdk-sqs": 3.224.0 - "@aws-sdk/middleware-serde": 3.224.0 - "@aws-sdk/middleware-signing": 3.224.0 - "@aws-sdk/middleware-stack": 3.224.0 - "@aws-sdk/middleware-user-agent": 3.224.0 - "@aws-sdk/node-config-provider": 3.224.0 - "@aws-sdk/node-http-handler": 3.224.0 - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/smithy-client": 3.224.0 - "@aws-sdk/types": 3.224.0 - "@aws-sdk/url-parser": 3.224.0 + "@aws-sdk/client-sts": 3.226.0 + "@aws-sdk/config-resolver": 3.226.0 + "@aws-sdk/credential-provider-node": 3.226.0 + "@aws-sdk/fetch-http-handler": 3.226.0 + "@aws-sdk/hash-node": 3.226.0 + "@aws-sdk/invalid-dependency": 3.226.0 + "@aws-sdk/md5-js": 3.226.0 + "@aws-sdk/middleware-content-length": 3.226.0 + "@aws-sdk/middleware-endpoint": 3.226.0 + "@aws-sdk/middleware-host-header": 3.226.0 + "@aws-sdk/middleware-logger": 3.226.0 + "@aws-sdk/middleware-recursion-detection": 3.226.0 + "@aws-sdk/middleware-retry": 3.226.0 + "@aws-sdk/middleware-sdk-sqs": 3.226.0 + "@aws-sdk/middleware-serde": 3.226.0 + "@aws-sdk/middleware-signing": 3.226.0 + "@aws-sdk/middleware-stack": 3.226.0 + "@aws-sdk/middleware-user-agent": 3.226.0 + "@aws-sdk/node-config-provider": 3.226.0 + "@aws-sdk/node-http-handler": 3.226.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/smithy-client": 3.226.0 + "@aws-sdk/types": 3.226.0 + "@aws-sdk/url-parser": 3.226.0 "@aws-sdk/util-base64": 3.208.0 "@aws-sdk/util-body-length-browser": 3.188.0 "@aws-sdk/util-body-length-node": 3.208.0 - "@aws-sdk/util-defaults-mode-browser": 3.224.0 - "@aws-sdk/util-defaults-mode-node": 3.224.0 - "@aws-sdk/util-endpoints": 3.224.0 - "@aws-sdk/util-user-agent-browser": 3.224.0 - "@aws-sdk/util-user-agent-node": 3.224.0 + "@aws-sdk/util-defaults-mode-browser": 3.226.0 + "@aws-sdk/util-defaults-mode-node": 3.226.0 + "@aws-sdk/util-endpoints": 3.226.0 + "@aws-sdk/util-user-agent-browser": 3.226.0 + "@aws-sdk/util-user-agent-node": 3.226.0 "@aws-sdk/util-utf8-browser": 3.188.0 "@aws-sdk/util-utf8-node": 3.208.0 fast-xml-parser: 4.0.11 tslib: ^2.3.1 - checksum: 71141166e3445627dd2bc9d194f6a4c87a4652e91ed79a2521297d1e72badba39495bb5c316050ac8a310c9422e063201d4d59c027b770995e8d6de8fa38d188 + checksum: 514fddd0abb2ec104bde0b08c206859a1ed1cc731996fb79c7ea5f4726ebb7329f5af827b3f0b9958be79c4e188e276cb81f211f6de4eae6bcf6e5f20f7bc447 languageName: node linkType: hard -"@aws-sdk/client-sso-oidc@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/client-sso-oidc@npm:3.224.0" +"@aws-sdk/client-sso-oidc@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/client-sso-oidc@npm:3.226.0" dependencies: "@aws-crypto/sha256-browser": 2.0.0 "@aws-crypto/sha256-js": 2.0.0 - "@aws-sdk/config-resolver": 3.224.0 - "@aws-sdk/fetch-http-handler": 3.224.0 - "@aws-sdk/hash-node": 3.224.0 - "@aws-sdk/invalid-dependency": 3.224.0 - "@aws-sdk/middleware-content-length": 3.224.0 - "@aws-sdk/middleware-endpoint": 3.224.0 - "@aws-sdk/middleware-host-header": 3.224.0 - "@aws-sdk/middleware-logger": 3.224.0 - "@aws-sdk/middleware-recursion-detection": 3.224.0 - "@aws-sdk/middleware-retry": 3.224.0 - "@aws-sdk/middleware-serde": 3.224.0 - "@aws-sdk/middleware-stack": 3.224.0 - "@aws-sdk/middleware-user-agent": 3.224.0 - "@aws-sdk/node-config-provider": 3.224.0 - "@aws-sdk/node-http-handler": 3.224.0 - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/smithy-client": 3.224.0 - "@aws-sdk/types": 3.224.0 - "@aws-sdk/url-parser": 3.224.0 + "@aws-sdk/config-resolver": 3.226.0 + "@aws-sdk/fetch-http-handler": 3.226.0 + "@aws-sdk/hash-node": 3.226.0 + "@aws-sdk/invalid-dependency": 3.226.0 + "@aws-sdk/middleware-content-length": 3.226.0 + "@aws-sdk/middleware-endpoint": 3.226.0 + "@aws-sdk/middleware-host-header": 3.226.0 + "@aws-sdk/middleware-logger": 3.226.0 + "@aws-sdk/middleware-recursion-detection": 3.226.0 + "@aws-sdk/middleware-retry": 3.226.0 + "@aws-sdk/middleware-serde": 3.226.0 + "@aws-sdk/middleware-stack": 3.226.0 + "@aws-sdk/middleware-user-agent": 3.226.0 + "@aws-sdk/node-config-provider": 3.226.0 + "@aws-sdk/node-http-handler": 3.226.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/smithy-client": 3.226.0 + "@aws-sdk/types": 3.226.0 + "@aws-sdk/url-parser": 3.226.0 "@aws-sdk/util-base64": 3.208.0 "@aws-sdk/util-body-length-browser": 3.188.0 "@aws-sdk/util-body-length-node": 3.208.0 - "@aws-sdk/util-defaults-mode-browser": 3.224.0 - "@aws-sdk/util-defaults-mode-node": 3.224.0 - "@aws-sdk/util-endpoints": 3.224.0 - "@aws-sdk/util-user-agent-browser": 3.224.0 - "@aws-sdk/util-user-agent-node": 3.224.0 + "@aws-sdk/util-defaults-mode-browser": 3.226.0 + "@aws-sdk/util-defaults-mode-node": 3.226.0 + "@aws-sdk/util-endpoints": 3.226.0 + "@aws-sdk/util-user-agent-browser": 3.226.0 + "@aws-sdk/util-user-agent-node": 3.226.0 "@aws-sdk/util-utf8-browser": 3.188.0 "@aws-sdk/util-utf8-node": 3.208.0 tslib: ^2.3.1 - checksum: 465f65014e007f85d331148607a5b51aa3bbbbb7a95705ed9daf5f562047ee190821e7cf6e84c6f4ae68ef8abe0ab2e84000b77a54757073210992d503fbe786 + checksum: 911e3484bba6b500da9817734851614d11f00ade21ad77d9ec722dc60f37ed1154dc12d37faf55ea425f750277fc361e71a39bddbda86c05fef0981db258bd1d languageName: node linkType: hard -"@aws-sdk/client-sso@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/client-sso@npm:3.224.0" +"@aws-sdk/client-sso@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/client-sso@npm:3.226.0" dependencies: "@aws-crypto/sha256-browser": 2.0.0 "@aws-crypto/sha256-js": 2.0.0 - "@aws-sdk/config-resolver": 3.224.0 - "@aws-sdk/fetch-http-handler": 3.224.0 - "@aws-sdk/hash-node": 3.224.0 - "@aws-sdk/invalid-dependency": 3.224.0 - "@aws-sdk/middleware-content-length": 3.224.0 - "@aws-sdk/middleware-endpoint": 3.224.0 - "@aws-sdk/middleware-host-header": 3.224.0 - "@aws-sdk/middleware-logger": 3.224.0 - "@aws-sdk/middleware-recursion-detection": 3.224.0 - "@aws-sdk/middleware-retry": 3.224.0 - "@aws-sdk/middleware-serde": 3.224.0 - "@aws-sdk/middleware-stack": 3.224.0 - "@aws-sdk/middleware-user-agent": 3.224.0 - "@aws-sdk/node-config-provider": 3.224.0 - "@aws-sdk/node-http-handler": 3.224.0 - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/smithy-client": 3.224.0 - "@aws-sdk/types": 3.224.0 - "@aws-sdk/url-parser": 3.224.0 + "@aws-sdk/config-resolver": 3.226.0 + "@aws-sdk/fetch-http-handler": 3.226.0 + "@aws-sdk/hash-node": 3.226.0 + "@aws-sdk/invalid-dependency": 3.226.0 + "@aws-sdk/middleware-content-length": 3.226.0 + "@aws-sdk/middleware-endpoint": 3.226.0 + "@aws-sdk/middleware-host-header": 3.226.0 + "@aws-sdk/middleware-logger": 3.226.0 + "@aws-sdk/middleware-recursion-detection": 3.226.0 + "@aws-sdk/middleware-retry": 3.226.0 + "@aws-sdk/middleware-serde": 3.226.0 + "@aws-sdk/middleware-stack": 3.226.0 + "@aws-sdk/middleware-user-agent": 3.226.0 + "@aws-sdk/node-config-provider": 3.226.0 + "@aws-sdk/node-http-handler": 3.226.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/smithy-client": 3.226.0 + "@aws-sdk/types": 3.226.0 + "@aws-sdk/url-parser": 3.226.0 "@aws-sdk/util-base64": 3.208.0 "@aws-sdk/util-body-length-browser": 3.188.0 "@aws-sdk/util-body-length-node": 3.208.0 - "@aws-sdk/util-defaults-mode-browser": 3.224.0 - "@aws-sdk/util-defaults-mode-node": 3.224.0 - "@aws-sdk/util-endpoints": 3.224.0 - "@aws-sdk/util-user-agent-browser": 3.224.0 - "@aws-sdk/util-user-agent-node": 3.224.0 + "@aws-sdk/util-defaults-mode-browser": 3.226.0 + "@aws-sdk/util-defaults-mode-node": 3.226.0 + "@aws-sdk/util-endpoints": 3.226.0 + "@aws-sdk/util-user-agent-browser": 3.226.0 + "@aws-sdk/util-user-agent-node": 3.226.0 "@aws-sdk/util-utf8-browser": 3.188.0 "@aws-sdk/util-utf8-node": 3.208.0 tslib: ^2.3.1 - checksum: dec6e779b6aef5a7bb1fec4ae8690c87a186c6dbb5401b557f885356f6f0298047ae89dd22115d0678ef4fbcb534636c895115c9fc1776d05683715d5328e595 + checksum: c0747f2e8611e3aa1b46a6a0c44e8ed53271853a43999ec7d94d228c962b88c0f7dd6a9b4c6fe10b0224e31a57d78787753635357fb014c7ceb9125a9eaaac38 languageName: node linkType: hard -"@aws-sdk/client-sts@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/client-sts@npm:3.224.0" +"@aws-sdk/client-sts@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/client-sts@npm:3.226.0" dependencies: "@aws-crypto/sha256-browser": 2.0.0 "@aws-crypto/sha256-js": 2.0.0 - "@aws-sdk/config-resolver": 3.224.0 - "@aws-sdk/credential-provider-node": 3.224.0 - "@aws-sdk/fetch-http-handler": 3.224.0 - "@aws-sdk/hash-node": 3.224.0 - "@aws-sdk/invalid-dependency": 3.224.0 - "@aws-sdk/middleware-content-length": 3.224.0 - "@aws-sdk/middleware-endpoint": 3.224.0 - "@aws-sdk/middleware-host-header": 3.224.0 - "@aws-sdk/middleware-logger": 3.224.0 - "@aws-sdk/middleware-recursion-detection": 3.224.0 - "@aws-sdk/middleware-retry": 3.224.0 - "@aws-sdk/middleware-sdk-sts": 3.224.0 - "@aws-sdk/middleware-serde": 3.224.0 - "@aws-sdk/middleware-signing": 3.224.0 - "@aws-sdk/middleware-stack": 3.224.0 - "@aws-sdk/middleware-user-agent": 3.224.0 - "@aws-sdk/node-config-provider": 3.224.0 - "@aws-sdk/node-http-handler": 3.224.0 - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/smithy-client": 3.224.0 - "@aws-sdk/types": 3.224.0 - "@aws-sdk/url-parser": 3.224.0 + "@aws-sdk/config-resolver": 3.226.0 + "@aws-sdk/credential-provider-node": 3.226.0 + "@aws-sdk/fetch-http-handler": 3.226.0 + "@aws-sdk/hash-node": 3.226.0 + "@aws-sdk/invalid-dependency": 3.226.0 + "@aws-sdk/middleware-content-length": 3.226.0 + "@aws-sdk/middleware-endpoint": 3.226.0 + "@aws-sdk/middleware-host-header": 3.226.0 + "@aws-sdk/middleware-logger": 3.226.0 + "@aws-sdk/middleware-recursion-detection": 3.226.0 + "@aws-sdk/middleware-retry": 3.226.0 + "@aws-sdk/middleware-sdk-sts": 3.226.0 + "@aws-sdk/middleware-serde": 3.226.0 + "@aws-sdk/middleware-signing": 3.226.0 + "@aws-sdk/middleware-stack": 3.226.0 + "@aws-sdk/middleware-user-agent": 3.226.0 + "@aws-sdk/node-config-provider": 3.226.0 + "@aws-sdk/node-http-handler": 3.226.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/smithy-client": 3.226.0 + "@aws-sdk/types": 3.226.0 + "@aws-sdk/url-parser": 3.226.0 "@aws-sdk/util-base64": 3.208.0 "@aws-sdk/util-body-length-browser": 3.188.0 "@aws-sdk/util-body-length-node": 3.208.0 - "@aws-sdk/util-defaults-mode-browser": 3.224.0 - "@aws-sdk/util-defaults-mode-node": 3.224.0 - "@aws-sdk/util-endpoints": 3.224.0 - "@aws-sdk/util-user-agent-browser": 3.224.0 - "@aws-sdk/util-user-agent-node": 3.224.0 + "@aws-sdk/util-defaults-mode-browser": 3.226.0 + "@aws-sdk/util-defaults-mode-node": 3.226.0 + "@aws-sdk/util-endpoints": 3.226.0 + "@aws-sdk/util-user-agent-browser": 3.226.0 + "@aws-sdk/util-user-agent-node": 3.226.0 "@aws-sdk/util-utf8-browser": 3.188.0 "@aws-sdk/util-utf8-node": 3.208.0 fast-xml-parser: 4.0.11 tslib: ^2.3.1 - checksum: 7515beb79d882711fed63b175a4095bb2b331c834ab5814d741358abe0b67e429d1faf8dfa86082527e673171db46b7596e99fa6eaeb4934eb3d3adc7d59e53d + checksum: df44bfa28034e6361fc300ce42350ad975afcfae833a59bf8b9ccee8f8f4cc606258b905fa34b052eeb71941d1d471ba89c9b6df7bc0188147038082626c1155 languageName: node linkType: hard -"@aws-sdk/config-resolver@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/config-resolver@npm:3.224.0" +"@aws-sdk/config-resolver@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/config-resolver@npm:3.226.0" dependencies: - "@aws-sdk/signature-v4": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/signature-v4": 3.226.0 + "@aws-sdk/types": 3.226.0 "@aws-sdk/util-config-provider": 3.208.0 - "@aws-sdk/util-middleware": 3.224.0 + "@aws-sdk/util-middleware": 3.226.0 tslib: ^2.3.1 - checksum: a36c7963a81c45651a89a7a6da90bd189b0c784cc4a60a149908bb5e9aac23da47235ef03ce9288240400a917392c69380dc61eb50934f9cf9dffa877ad821c1 + checksum: 08bfbc0528da98cba2b46bda05cd9af46f6eced3dbe5e415b11465501fe672d99b869adc9f08cfd2f138c9b61ac119b43eb9a90a490395aec705e1b795022d4d languageName: node linkType: hard -"@aws-sdk/credential-provider-cognito-identity@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/credential-provider-cognito-identity@npm:3.224.0" +"@aws-sdk/credential-provider-cognito-identity@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/credential-provider-cognito-identity@npm:3.226.0" dependencies: - "@aws-sdk/client-cognito-identity": 3.224.0 - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/client-cognito-identity": 3.226.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: fb7d6217073e3795133dbf1130e7029037895b7387e2c3784991e009579e6af3a2e7d11f6ea6d0bf5d0ef1aab7753123125dfacc40be59922afb245d6ab60a0f + checksum: 13d6a35407600ef40c8e02ae93c95d40aa0c0427846bdcbb6fc0d6166ab103aab1e51900d9cd29c8cc272fa6a1ea1699bf4e1c7961f6e62d0b6c376527b4b5ac languageName: node linkType: hard -"@aws-sdk/credential-provider-env@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/credential-provider-env@npm:3.224.0" +"@aws-sdk/credential-provider-env@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/credential-provider-env@npm:3.226.0" dependencies: - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 56e78345184467d56fc2689cacc57631d63b67381a7cf6a25f349d1ea7de7aa850166dc1bf91bc290c6709ec0e2a72b8baa16a9fb59450c168f9520265b3eb9d + checksum: 4f764d8d07ba1f65503ed2440ebfbccf98c141b11a087a582ea606d06bc764c91641b4ec505ab0d4750025a01128b041d9514a3ec868c385f038b5373b513293 languageName: node linkType: hard -"@aws-sdk/credential-provider-imds@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/credential-provider-imds@npm:3.224.0" +"@aws-sdk/credential-provider-imds@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/credential-provider-imds@npm:3.226.0" dependencies: - "@aws-sdk/node-config-provider": 3.224.0 - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/types": 3.224.0 - "@aws-sdk/url-parser": 3.224.0 + "@aws-sdk/node-config-provider": 3.226.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/types": 3.226.0 + "@aws-sdk/url-parser": 3.226.0 tslib: ^2.3.1 - checksum: b33c6dc901aa74c4105df4d6c8e20f8f27c0c8e6875c4a1b3a70719a844beb7a3b280b11652cc746cc89090238645ec67a4e336c59d1fc753e95621bca166b0b + checksum: c0480ca127e6715ebf98ff41a8ea47d139166b8632582782df46fc9e568ff579cfc0768af80082ae6eea3eb6750edb41c4a4c39281b777c578fdb7f8ef8b641c languageName: node linkType: hard -"@aws-sdk/credential-provider-ini@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/credential-provider-ini@npm:3.224.0" +"@aws-sdk/credential-provider-ini@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/credential-provider-ini@npm:3.226.0" dependencies: - "@aws-sdk/credential-provider-env": 3.224.0 - "@aws-sdk/credential-provider-imds": 3.224.0 - "@aws-sdk/credential-provider-sso": 3.224.0 - "@aws-sdk/credential-provider-web-identity": 3.224.0 - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/shared-ini-file-loader": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/credential-provider-env": 3.226.0 + "@aws-sdk/credential-provider-imds": 3.226.0 + "@aws-sdk/credential-provider-sso": 3.226.0 + "@aws-sdk/credential-provider-web-identity": 3.226.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/shared-ini-file-loader": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 7f3f0bfe4fddbfd65a9ea694b640437dea15afd9f6449206a79dad51876c7d3fc9e93b8a822bdfec7c91390ebf100aee8ec69da68260ff95ae5ac0a7ad94e334 + checksum: ed1a0db8eddaeb6d8ad181c11c317c1e2c53061d03444252d5d142897e02c655848728f00b356fce7799bffacacb40a4ca0680098b904cfc798914206ef60290 languageName: node linkType: hard -"@aws-sdk/credential-provider-node@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/credential-provider-node@npm:3.224.0" +"@aws-sdk/credential-provider-node@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/credential-provider-node@npm:3.226.0" dependencies: - "@aws-sdk/credential-provider-env": 3.224.0 - "@aws-sdk/credential-provider-imds": 3.224.0 - "@aws-sdk/credential-provider-ini": 3.224.0 - "@aws-sdk/credential-provider-process": 3.224.0 - "@aws-sdk/credential-provider-sso": 3.224.0 - "@aws-sdk/credential-provider-web-identity": 3.224.0 - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/shared-ini-file-loader": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/credential-provider-env": 3.226.0 + "@aws-sdk/credential-provider-imds": 3.226.0 + "@aws-sdk/credential-provider-ini": 3.226.0 + "@aws-sdk/credential-provider-process": 3.226.0 + "@aws-sdk/credential-provider-sso": 3.226.0 + "@aws-sdk/credential-provider-web-identity": 3.226.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/shared-ini-file-loader": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: b9a8112e8c9bd9ab6db0600119380deda6cf7b90accd29be8ad8e37a6a54feb65a69be5e2211f698d2fd5006df09e3c6124b70203402c5c297736dc4856d4298 + checksum: e015d97c354743e91855729b05db80926da8c16e3170c89699f6519b5911ea4ccd95822ecce6633315e08310c59aa374bb90e56764920241cfffb0b736eda432 languageName: node linkType: hard -"@aws-sdk/credential-provider-process@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/credential-provider-process@npm:3.224.0" +"@aws-sdk/credential-provider-process@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/credential-provider-process@npm:3.226.0" dependencies: - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/shared-ini-file-loader": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/shared-ini-file-loader": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: ca2ed6dcea75cdbcd625e374b52d8801be551f460babe6eea7f9137cef6dc576621715fdf92b344689c087dfd0922ae1a6f821a49284bf84435fcf5fd4503022 + checksum: 3ad24e2784ae7929f8932a187e0a5a06fc62d31ccb393fd2d9fa86d75c1109809c96b69edb17c3d7bd7d087b97662818d5320742a4cd70add06999a3b2ec3a53 languageName: node linkType: hard -"@aws-sdk/credential-provider-sso@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/credential-provider-sso@npm:3.224.0" +"@aws-sdk/credential-provider-sso@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/credential-provider-sso@npm:3.226.0" dependencies: - "@aws-sdk/client-sso": 3.224.0 - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/shared-ini-file-loader": 3.224.0 - "@aws-sdk/token-providers": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/client-sso": 3.226.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/shared-ini-file-loader": 3.226.0 + "@aws-sdk/token-providers": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: b683443ddc87581a61a0605e2cf656105c17df22ee8d9d7980c8e82cf3b91f5fb146962d6339f2c979b53a20442daeec210387b0f836cd4fc8ab1e789605d242 + checksum: 1fc6631926569ab17cc5f69cfa836b35a0f985bee6e760af46892961547e1f8f7376955ab8f404c1aa1e9c3edffe5965b5fa4feb09b2b791694415c4dea26fec languageName: node linkType: hard -"@aws-sdk/credential-provider-web-identity@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/credential-provider-web-identity@npm:3.224.0" +"@aws-sdk/credential-provider-web-identity@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/credential-provider-web-identity@npm:3.226.0" dependencies: - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: bbb4617981743074b50e99c73861a6f2ef89725cdfaa2f6dece73317d0d8baa007edf5ce001d6efdf5707da97b90fb39c9049a25f4fc57547329d1d0388ccc8d + checksum: f536d9af3d900eee31b6e7966ff827713fd5f661ae477f0911eebdac3e544eb5a732df0be0c9162058943e0aeb89333c97a1758236e8f9760c5e87280fd926fa languageName: node linkType: hard "@aws-sdk/credential-providers@npm:^3.208.0": - version: 3.224.0 - resolution: "@aws-sdk/credential-providers@npm:3.224.0" + version: 3.226.0 + resolution: "@aws-sdk/credential-providers@npm:3.226.0" dependencies: - "@aws-sdk/client-cognito-identity": 3.224.0 - "@aws-sdk/client-sso": 3.224.0 - "@aws-sdk/client-sts": 3.224.0 - "@aws-sdk/credential-provider-cognito-identity": 3.224.0 - "@aws-sdk/credential-provider-env": 3.224.0 - "@aws-sdk/credential-provider-imds": 3.224.0 - "@aws-sdk/credential-provider-ini": 3.224.0 - "@aws-sdk/credential-provider-node": 3.224.0 - "@aws-sdk/credential-provider-process": 3.224.0 - "@aws-sdk/credential-provider-sso": 3.224.0 - "@aws-sdk/credential-provider-web-identity": 3.224.0 - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/shared-ini-file-loader": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/client-cognito-identity": 3.226.0 + "@aws-sdk/client-sso": 3.226.0 + "@aws-sdk/client-sts": 3.226.0 + "@aws-sdk/credential-provider-cognito-identity": 3.226.0 + "@aws-sdk/credential-provider-env": 3.226.0 + "@aws-sdk/credential-provider-imds": 3.226.0 + "@aws-sdk/credential-provider-ini": 3.226.0 + "@aws-sdk/credential-provider-node": 3.226.0 + "@aws-sdk/credential-provider-process": 3.226.0 + "@aws-sdk/credential-provider-sso": 3.226.0 + "@aws-sdk/credential-provider-web-identity": 3.226.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/shared-ini-file-loader": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 584dedd2d444c7d16923df3ded5e94fa2c74732cb1f70f32aa967696b941810df55941c3e9e792bf885b8980fea45763c77470ab97736ecb02c30da90e98a819 + checksum: 6e034e36c25df64e08e1db1883dc71e1f2cda4b3ffdc61af60a0482a5e7d1493fa067b0897bd59d7a6e31a45e33515d2fce7545007a0d346b3c74cc3bcc69523 languageName: node linkType: hard -"@aws-sdk/eventstream-codec@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/eventstream-codec@npm:3.224.0" +"@aws-sdk/eventstream-codec@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/eventstream-codec@npm:3.226.0" dependencies: "@aws-crypto/crc32": 2.0.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 "@aws-sdk/util-hex-encoding": 3.201.0 tslib: ^2.3.1 - checksum: 622758cc6968e04a37017be40a28a2febe75444752e84c4f67568fe5ca7f7272f831b3231d3bb8be7af5d81825999270ccb02c77ced47bcfbcbf2e18c5af8381 + checksum: 3b3a584c02bcf6cfe35e2ed562e8d82844a51c1efd2f9d79a72ecf60ba9a63a51c568c8b5a49cdfb6c49fe655be0454c082098285378c66494109dbf7b5bb28f languageName: node linkType: hard -"@aws-sdk/eventstream-serde-browser@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/eventstream-serde-browser@npm:3.224.0" +"@aws-sdk/eventstream-serde-browser@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/eventstream-serde-browser@npm:3.226.0" dependencies: - "@aws-sdk/eventstream-serde-universal": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/eventstream-serde-universal": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 68dc85dad5d2447b7f3f9ff900917ff27b471ce8598c624e0e9b5f571b0cc708bffae5b634431d990c4c079069a94371611003849bfda01660fdb87b0a84e2e0 + checksum: 12530898d3debe33fbc73c6b68c7a889e4c3e29c183180d9d012e9b3761af1fd58feaa9a6e8069ccf380442c7d71ad6f36d5f0af078ae6d29af944ca6f4a9610 languageName: node linkType: hard -"@aws-sdk/eventstream-serde-config-resolver@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/eventstream-serde-config-resolver@npm:3.224.0" +"@aws-sdk/eventstream-serde-config-resolver@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/eventstream-serde-config-resolver@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 6fff16c3cb290131c35e933024e51d58d9e3a5c8ec02e5fde00e9c798755cffc97fe88ea3606cc4879b03acb9b639446fa036ba5b65f1a644226891635ae7f90 + checksum: 2438bfd3bf00e401caa79c2b874be6ecfac24db74f3ec9e1064a3d5c09a7cfe0fa9c0086e30ee3bc8853f550efc6c6ecb2936caafd238478870a803863fbea23 languageName: node linkType: hard -"@aws-sdk/eventstream-serde-node@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/eventstream-serde-node@npm:3.224.0" +"@aws-sdk/eventstream-serde-node@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/eventstream-serde-node@npm:3.226.0" dependencies: - "@aws-sdk/eventstream-serde-universal": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/eventstream-serde-universal": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: de4f0859d29271c18ca9c971f61e1e2997b26171537ce39dab0fd7588c96fc1bf8e1e7ba21c47643a2e22aaba0558172cfa4fea32e9ad9c83c29c52f33a8711a + checksum: 89c2154fb4bb544f87e5fdda37cc99f724125892417d78763a98a0a4390b2ba89e8a57521a57b213827688ce8d8bc14971ae87f1137cc26d9f3ebbae142d5a96 languageName: node linkType: hard -"@aws-sdk/eventstream-serde-universal@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/eventstream-serde-universal@npm:3.224.0" +"@aws-sdk/eventstream-serde-universal@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/eventstream-serde-universal@npm:3.226.0" dependencies: - "@aws-sdk/eventstream-codec": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/eventstream-codec": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: f59d9b5a032ea5c7d5d44dedae49b7a2082b230cf9c959c691cddecf49e526a8c30ac3dcab0caa51e92cbc4bf733401c50f8883ea477f8c4549f02a4be457464 + checksum: da1eea2d8b0f6972a5016ca66b2a8da0e2e72b8449a661c8cf4678894e22bcef20d5c1534c5c5a22b4c7d1d242bf894b7220c8d712123e092efa8c5fa8aca02f languageName: node linkType: hard @@ -894,59 +894,59 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/fetch-http-handler@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/fetch-http-handler@npm:3.224.0" +"@aws-sdk/fetch-http-handler@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/fetch-http-handler@npm:3.226.0" dependencies: - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/querystring-builder": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/querystring-builder": 3.226.0 + "@aws-sdk/types": 3.226.0 "@aws-sdk/util-base64": 3.208.0 tslib: ^2.3.1 - checksum: 9f259ff8f43cc2489e3e56680ff6ebe83a73dff68c516041c950652435a3f04ecf188c916b0fd707519d58f5ddb6bdc6d1ba08defc480fd824a2da4853f69efd + checksum: f12ed12088aee05e4c6a742356017f55f26fec740c12c4d89c4eb283203171d032d99c9fa45fb28cc7798b35410cc55736df1c78d8eea014667e489f1b276f3a languageName: node linkType: hard -"@aws-sdk/hash-blob-browser@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/hash-blob-browser@npm:3.224.0" +"@aws-sdk/hash-blob-browser@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/hash-blob-browser@npm:3.226.0" dependencies: "@aws-sdk/chunked-blob-reader": 3.188.0 "@aws-sdk/chunked-blob-reader-native": 3.208.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 095ce6321cce2543315191b9add1e4d80c5d0ff0acdf33eebb286dbfbb66253611ddeddfb7d24eaba80279a6a946360bdee7e9f09189d786fd9684991f3aec7d + checksum: 98fecc0646a37bfb541a2e8bd1da27f0b208f9d51e6efc0e2e9e833cb1872a4205c5b47568582805a3e98240f3fe2f16a5aaee6693d1e1589fd6918afce554b0 languageName: node linkType: hard -"@aws-sdk/hash-node@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/hash-node@npm:3.224.0" +"@aws-sdk/hash-node@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/hash-node@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 "@aws-sdk/util-buffer-from": 3.208.0 tslib: ^2.3.1 - checksum: 4bc0887de1572a482b90047e8abe3ef82381535702dc59679c976cb31ae6cd3b784458475618ea701e7b19876708c9fe41356229aed38355123672fcdc1050fa + checksum: cbe466e2e3efdbb3aefe7772da734f6d00e0d1572c60e497b09460e055093b155c5e94bd3718135238bc05e58cb49abcdcd0fe5e34458bd3563b89d3d4a42251 languageName: node linkType: hard -"@aws-sdk/hash-stream-node@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/hash-stream-node@npm:3.224.0" +"@aws-sdk/hash-stream-node@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/hash-stream-node@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: db411e1ff7aca6dae01abe9400e95ad6f6fd5108deeb28c29d5f619f0bda19e25851ad511b8e0d35a304f24b61cee597d18b949cc17776402386bf784b4f264a + checksum: a6acb592e815fa48b39e7917599405ebf77c8c03e42ef039eea8886e0bda98c4d8a963506b137514787dca65cf4594cf756d49adcc67b0777b71f2ce7c4cd0da languageName: node linkType: hard -"@aws-sdk/invalid-dependency@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/invalid-dependency@npm:3.224.0" +"@aws-sdk/invalid-dependency@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/invalid-dependency@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: b356eac51b083580d9337be21132c276e08c7b86304619af5bb84c7aa893dee8928b76d730ce854314527e97d4140c99c3bf96960c721d2f5d4c716fa8475a39 + checksum: 4a6168b50675881442c9bc87e8f27e8317ece9251fbfd3d812d32edcd83f687328e6de518fa9f11b1bc984ad3a3454cd43b59c0362f04c03687ce5250ac5d720 languageName: node linkType: hard @@ -960,11 +960,11 @@ __metadata: linkType: hard "@aws-sdk/lib-storage@npm:^3.208.0": - version: 3.224.0 - resolution: "@aws-sdk/lib-storage@npm:3.224.0" + version: 3.226.0 + resolution: "@aws-sdk/lib-storage@npm:3.226.0" dependencies: - "@aws-sdk/middleware-endpoint": 3.224.0 - "@aws-sdk/smithy-client": 3.224.0 + "@aws-sdk/middleware-endpoint": 3.226.0 + "@aws-sdk/smithy-client": 3.226.0 buffer: 5.6.0 events: 3.3.0 stream-browserify: 3.0.0 @@ -972,267 +972,267 @@ __metadata: peerDependencies: "@aws-sdk/abort-controller": ^3.0.0 "@aws-sdk/client-s3": ^3.0.0 - checksum: fe23a8adce2df28cc46b941010c7a426d86f3c6993def09d3c4a2d1e14400f0f511b8e15caaa10cb64cb1863413cc6dd026750cc4195b246ee3b6976b3df9a18 + checksum: d04ffdd8d4baad0727a4d7f832bc57d05ce733ee97c96003bb160f92ca1af3ce575e3888a0f6e2e05d972ee766802fdf8af884361a5f864920f280e967a1a2cf languageName: node linkType: hard -"@aws-sdk/md5-js@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/md5-js@npm:3.224.0" +"@aws-sdk/md5-js@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/md5-js@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 "@aws-sdk/util-utf8-browser": 3.188.0 "@aws-sdk/util-utf8-node": 3.208.0 tslib: ^2.3.1 - checksum: 6c886d29a47da9af5e71b637e7adbc28f4e09cbb69d31ec2f69f2d9cdb74288fa82bb24e46bceeee49c0bb4d855b2cf2f35951a7920ac507a0776a49907ad5e4 + checksum: 7d2dc4fefaf3136bf30e22f4645588624c8e2696b638357676f18df3ea1c11a7fd8dddc5b50cae739383590b2d3e10be02fd033c764c57c882b5a2230edb2d6f languageName: node linkType: hard -"@aws-sdk/middleware-bucket-endpoint@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-bucket-endpoint@npm:3.224.0" +"@aws-sdk/middleware-bucket-endpoint@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-bucket-endpoint@npm:3.226.0" dependencies: - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/types": 3.226.0 "@aws-sdk/util-arn-parser": 3.208.0 "@aws-sdk/util-config-provider": 3.208.0 tslib: ^2.3.1 - checksum: 6653f5746286103e2b14278f204aac47336a924e410506cff7f51ce7450bd725f52414bd55c7f0bd201469ee520583c0d2091df3234cc05dc71d31ef54b71810 + checksum: bb5e4a8bd79d578c103d216274a8175c42cea06c138338fa0854a30b8eab0c6a14d5996814f2ac9dcc82e1f137a240e657bb1e8841b52d8ade5c275771c43ca5 languageName: node linkType: hard -"@aws-sdk/middleware-content-length@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-content-length@npm:3.224.0" +"@aws-sdk/middleware-content-length@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-content-length@npm:3.226.0" dependencies: - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 509fc34b438d2bc9a1fa0c987b41cfcff57b151e23b1d74b9b4d8c8533b99b9cd76b15cae845b8749e9eb33fb41e7f15813c4e9328a563acf2f7427ecb17d3b6 + checksum: 538b0222e2c8be61dc3090dfefe4b9ec1f0cbee155125edf5b654dbe36ab0b2d6befe3da6008194dc26bbf94d53b7e01edca683720061c4b43c6beb9dc278002 languageName: node linkType: hard -"@aws-sdk/middleware-endpoint@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-endpoint@npm:3.224.0" +"@aws-sdk/middleware-endpoint@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-endpoint@npm:3.226.0" dependencies: - "@aws-sdk/middleware-serde": 3.224.0 - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/signature-v4": 3.224.0 - "@aws-sdk/types": 3.224.0 - "@aws-sdk/url-parser": 3.224.0 + "@aws-sdk/middleware-serde": 3.226.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/signature-v4": 3.226.0 + "@aws-sdk/types": 3.226.0 + "@aws-sdk/url-parser": 3.226.0 "@aws-sdk/util-config-provider": 3.208.0 - "@aws-sdk/util-middleware": 3.224.0 + "@aws-sdk/util-middleware": 3.226.0 tslib: ^2.3.1 - checksum: ec64a4cd3fe5f8008a6783c26a41c31ddc8fc100c399d87e1fd3e9c13a8916debdc0d698ecb53fdee853a877bd026df4b40106e74b431510d392e619143d8dae + checksum: 22df6fd90e3b7d3edd58f427ebfbb229181ef190b2d795e0b400f70f95a573dc59666a08b26613db6bb1b764b13ba7ddec15fced2641e9678fad5500fca289be languageName: node linkType: hard -"@aws-sdk/middleware-expect-continue@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-expect-continue@npm:3.224.0" +"@aws-sdk/middleware-expect-continue@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-expect-continue@npm:3.226.0" dependencies: - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: b1ea74285d5b25538971a6fdf2c8836f6c0324b5f0fbf8863df7e8faf3d786612671a3e6694369ed4777256fc4bf920087706bf7916650128657be22224c4dc4 + checksum: 22924f008a7594cedde2c1f09b9b8ce005cf73bfcbf40330a2b783e42dffcd1fc88fd3a7b2b13da1ec4895c649f7605acea7b62dbbd3a45b00630b1cb9579bc4 languageName: node linkType: hard -"@aws-sdk/middleware-flexible-checksums@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-flexible-checksums@npm:3.224.0" +"@aws-sdk/middleware-flexible-checksums@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-flexible-checksums@npm:3.226.0" dependencies: "@aws-crypto/crc32": 2.0.0 "@aws-crypto/crc32c": 2.0.0 "@aws-sdk/is-array-buffer": 3.201.0 - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: cd8a7a9bedfe957e0167e812ce0b3b1410571272d955d272493baae64ac81166fd62ce8d72affa6d5e989728919515221e8c7733a32e53e1be6e61dbaa896597 + checksum: c4f0531b3c917a0bfaf3b877f7f15de30f5e7e0a8366206c680efe4d0f60a0e54d7861ee14d213f713e157f164283d51724288b16fba67fef39d628324fef588 languageName: node linkType: hard -"@aws-sdk/middleware-host-header@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-host-header@npm:3.224.0" +"@aws-sdk/middleware-host-header@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-host-header@npm:3.226.0" dependencies: - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 33c7e89226e07068413503b89d5f79adb677e3f80f55b5613089a35b7a9be5506d5bead56eacffd9bb658d2303d1224acd6037e8c563c794e45d2653664044e9 + checksum: 35eb7d9f5eb2e45c58bb5887a12dc80f22e8fc630d53d356064c62ce3354c9054717697168ab3818f14c7baae3246480aa7251e6fcaf5c7104498df5dca9bbd6 languageName: node linkType: hard -"@aws-sdk/middleware-location-constraint@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-location-constraint@npm:3.224.0" +"@aws-sdk/middleware-location-constraint@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-location-constraint@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: f8389db2f32fe8f30142e30a7eb0668a29a9825f009d031bd1b66c1360025594ffa785eb610c3a7f456809a77a67663654c5d3ebe6ed1563347b8ff9a0dcb618 + checksum: 31db89d8935771aa73552fd99bc3a8fb3cca25c4e5bf978e96d05fd3aa8ab8403774ddeee08f83a4c9e5b4630082001e5d13aec1924353257fa87184dffba3e1 languageName: node linkType: hard -"@aws-sdk/middleware-logger@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-logger@npm:3.224.0" +"@aws-sdk/middleware-logger@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-logger@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 0b859bcbcce1a5fb449d2be7180a08b27d45d634fe71f9ac7e999bfa7fca8e9d26cf47bfdadb02b499b7a7e65d8d83ba0e2d58b2fdd970266220f54fa7efc3a4 + checksum: 7b7c2eb336f376f6e8c50ebd0fd9b6e9a749e21d6f5c433683a4e98edb311bb76fafc0bb0abfd75f61001466879be7d68aefb3958a332e540160a93ee9371b44 languageName: node linkType: hard -"@aws-sdk/middleware-recursion-detection@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-recursion-detection@npm:3.224.0" +"@aws-sdk/middleware-recursion-detection@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-recursion-detection@npm:3.226.0" dependencies: - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 27649af44cc8b97a75e70796e816b467931316cc7c5d197cb66a75d61e59941bfc23508a480f796da537ca083fe977b000446327e4d2acd5d85810be70a997c5 + checksum: 194161d76a815df623c6699f7a7016268b4a01276baa3527682504001a4e2e948e2dd389d2065f62fe57509f20b9680ea907bfa52b861122335d276c9ab4ed61 languageName: node linkType: hard -"@aws-sdk/middleware-retry@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-retry@npm:3.224.0" +"@aws-sdk/middleware-retry@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-retry@npm:3.226.0" dependencies: - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/service-error-classification": 3.224.0 - "@aws-sdk/types": 3.224.0 - "@aws-sdk/util-middleware": 3.224.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/service-error-classification": 3.226.0 + "@aws-sdk/types": 3.226.0 + "@aws-sdk/util-middleware": 3.226.0 tslib: ^2.3.1 uuid: ^8.3.2 - checksum: a5d3b0db4a8734a3b693b2176a196c4f16101d911d8d2bd393b11f184f5335102cc9b5fec7ea4470784de12d2ef3e7c53be7e4b55c545483385fe7c6e33dd6c0 + checksum: 08da2eabddb6400a71d07f4e59dfe32d763089f070c42ab6ffc41a4028f05561efa974d19cf45c3e9227f79aea98175222ce9892941889a43939fecc1a39f48b languageName: node linkType: hard -"@aws-sdk/middleware-sdk-s3@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-sdk-s3@npm:3.224.0" +"@aws-sdk/middleware-sdk-s3@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-sdk-s3@npm:3.226.0" dependencies: - "@aws-sdk/middleware-bucket-endpoint": 3.224.0 - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/middleware-bucket-endpoint": 3.226.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/types": 3.226.0 "@aws-sdk/util-arn-parser": 3.208.0 tslib: ^2.3.1 - checksum: 872b249cb6c37d05da8352380ddbbe7cf841f7f7eebc46a7de69990dc124ce2ebab1221d9120f4a4870c74791817f14388e89f76a3817273c0750e2e41ea71dc + checksum: 61902d5378ead403f91c55a7156571747f2163189790ee0f59a2200f5cacf55af2567c20373773c803a82578f371ab82b7368bc42c08fba1349b6feb99ee8c2c languageName: node linkType: hard -"@aws-sdk/middleware-sdk-sqs@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-sdk-sqs@npm:3.224.0" +"@aws-sdk/middleware-sdk-sqs@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-sdk-sqs@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 "@aws-sdk/util-hex-encoding": 3.201.0 tslib: ^2.3.1 - checksum: 1c616b94b36b2f3cedf6c7b781785314bd8f59abbb9cb16596bea66b6448f65a5023a21121cf40889507227e892be1628adf9b290d24c0b1e5bd0718ff1993cf + checksum: 3c3d4cfaec156ad559a2409a095e7238e9fb6c1c6cb8b5304a62b61ea6b4241df907ea4ff0b71d0f0865b8338731113946e5df8cff361f73d2725730b2aa6db0 languageName: node linkType: hard -"@aws-sdk/middleware-sdk-sts@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-sdk-sts@npm:3.224.0" +"@aws-sdk/middleware-sdk-sts@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-sdk-sts@npm:3.226.0" dependencies: - "@aws-sdk/middleware-signing": 3.224.0 - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/signature-v4": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/middleware-signing": 3.226.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/signature-v4": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: a79b9522fbda47da0908e1e0d233205a8cb3ca917cc3d93e7a51ae4631a53178e923eac3d68cc025fa1aea66637f4b29f5437ad83b954ef94d1c03e9d3c41e60 + checksum: 0b6e11889c9f1b5264eb38288dea5bd2987fa28338651939d005eaa0524f9cd41c570c545b34b08d25d858397b3f14a2b6a21238a8c88fc08719ed50b835cb15 languageName: node linkType: hard -"@aws-sdk/middleware-serde@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-serde@npm:3.224.0" +"@aws-sdk/middleware-serde@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-serde@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 87d78fdc57003022844cf4aa9ba94ba0a9329e9aef927a1fedf5291e894f21f070815d7480aafdcf8fb393f419899fc9c7c41f17550ea5c02ec725f46b17e0ba + checksum: e8315079cc3ed4527d1434eb28becfaf951338d2ced21116753b52d3330eabbfd13f24382b3378bcc2c09ee0653a65e0d8353e9e32f512b69929908272b83fa2 languageName: node linkType: hard -"@aws-sdk/middleware-signing@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-signing@npm:3.224.0" +"@aws-sdk/middleware-signing@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-signing@npm:3.226.0" dependencies: - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/signature-v4": 3.224.0 - "@aws-sdk/types": 3.224.0 - "@aws-sdk/util-middleware": 3.224.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/signature-v4": 3.226.0 + "@aws-sdk/types": 3.226.0 + "@aws-sdk/util-middleware": 3.226.0 tslib: ^2.3.1 - checksum: 2730138c34e1bb3b1fc303b8056803060632dec90e5e9e036189b26555c2139f6de5a5a1c9ae5289de3da6a65f0de4f1e8661703a9c752f794b3d42b554d090a + checksum: 4f66483d359e21d1f7e56a1900f7271a561801e1ee4380deaa006ce750ce73245a5fe554d4c1ab60cf9ca127c41f35c53c0817c375df6f8f068c9ed7f9572444 languageName: node linkType: hard -"@aws-sdk/middleware-ssec@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-ssec@npm:3.224.0" +"@aws-sdk/middleware-ssec@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-ssec@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: d8f96f4b7dcc74f7871e646e3b6808b305e6f4b61401523045188e7b0c23781d28f4d0cace8bcbaf5e38f9c118f4cd7fe67304c1f80b57cab5b3833f5b20a53a + checksum: b85bf0e8be723a28a03dd7375284a5afc00edb04bf333a44286042768391243670b61b78361883d7c7890b118e0dc995219d7870227f845ac868a756e8a76a70 languageName: node linkType: hard -"@aws-sdk/middleware-stack@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-stack@npm:3.224.0" +"@aws-sdk/middleware-stack@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-stack@npm:3.226.0" dependencies: tslib: ^2.3.1 - checksum: 144e2b7e5aca6d1ad562bd9d1cba0cf6f12bf63a52dfef04ff38f32e76a76d2e4180fee6a81cc8c65ca2cef4c2617cabd74f360336f72e07068af9205536ebc4 + checksum: fd284bc28ee6ee576e3fe9dd3c2c9d64b8c95fa1bb213b5e41e23a57bbc2693652ee51028414eb278918235e591ef729db572b160bff963c6f74e7d6c15d33d0 languageName: node linkType: hard -"@aws-sdk/middleware-user-agent@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/middleware-user-agent@npm:3.224.0" +"@aws-sdk/middleware-user-agent@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/middleware-user-agent@npm:3.226.0" dependencies: - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: b12568290e2e4c367e387f3736eab1b442e7d16690ccc27044f25078dd27db4c1cc5635d3b76d5445545164d1184a4c38875f76f4461e0ea629b76df3f9ee03d + checksum: 8d3724aecd7b7f9f1a5b081263cb85864c969144105819f371a9675ee4f680c11e3b5ecbe181f57d2bfdde9c98b90267bd4afe0fc0822863041e0df52671f043 languageName: node linkType: hard -"@aws-sdk/node-config-provider@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/node-config-provider@npm:3.224.0" +"@aws-sdk/node-config-provider@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/node-config-provider@npm:3.226.0" dependencies: - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/shared-ini-file-loader": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/shared-ini-file-loader": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 26a970b6851d222b0cf7dcd5903c6b8a62c1092ae9ac414828891f47e12d32c84cf2a9f8677717d2c0126c4e3b4dbac2c834971de26d548e5c04b5b66b291cb9 + checksum: 0e7c4c7ea20c5361c71f1ab0b99c19964f76c187659714cd6e87e1cc264d0ab29965d33618d9d47d373e3b108558cd6628fb36a00cdac614849e5044c6776213 languageName: node linkType: hard -"@aws-sdk/node-http-handler@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/node-http-handler@npm:3.224.0" +"@aws-sdk/node-http-handler@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/node-http-handler@npm:3.226.0" dependencies: - "@aws-sdk/abort-controller": 3.224.0 - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/querystring-builder": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/abort-controller": 3.226.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/querystring-builder": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 14ea9f596919cd6b577e9dac59131aea5ec3651ed3ad45a4010ac46ec240a3b608116b0da0fe5d2c945bfb4015630592bb1ccea1736513d4af070c6225f357be + checksum: e4080ec9f859be4b191416084fef8d2687f24d973169b8fef592a867c23c14fad74f4bea20f451b98fedfee6a265f2c89b0bee4fe7aabdb1fd05a7475bf12b91 languageName: node linkType: hard -"@aws-sdk/property-provider@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/property-provider@npm:3.224.0" +"@aws-sdk/property-provider@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/property-provider@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 314c35b9e2b721f996a3098464fd3af306fe41c68730c7ae22e50f16896ec5b680540567a2efa4c94fa0601d521a4070ae61e1781d7e9210cd0cf934f281fe60 + checksum: d63c53313bb2797ab2ecfa2cff827b63417c0cc31abe8c666cd018a1d70da877b61ef1fb21e504d6408e72b20544756089938b44869b003ba3c313b39bca6d45 languageName: node linkType: hard @@ -1246,13 +1246,13 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/protocol-http@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/protocol-http@npm:3.224.0" +"@aws-sdk/protocol-http@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/protocol-http@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 148d8884566823a7bec10b2bf85c123891ddd23b9e1a544e60a9f55902e390a469699964ddfe19810883ca8a72170918a0a57415e8dfe1da0ed26162d8dedb05 + checksum: 92cf35e1026a812c6c7ce8309fd685162ab7796237fc063e71dd9b292abf25902d6c0e02e93219101e1f506ef68a98f02eed6a61b18b8acd450ecb4bf8c1b467 languageName: node linkType: hard @@ -1267,51 +1267,51 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/querystring-builder@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/querystring-builder@npm:3.224.0" +"@aws-sdk/querystring-builder@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/querystring-builder@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 "@aws-sdk/util-uri-escape": 3.201.0 tslib: ^2.3.1 - checksum: 6435f62feb5bfa478a4f40b7bf4525de8cf021a69b05dd73951eb03c6bea610477942b5c79beca5c4f9c0544948a105c4ef38eb0470b8266cec9c2d656047c68 + checksum: 0014b8876d402787c56f606f66e79eb48ff100b01d7a657bb0766a7723ca184da0dd2aed405837e7e6c5781fd868d7a903eef25884b2d1881d30e2454f5c2bfc languageName: node linkType: hard -"@aws-sdk/querystring-parser@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/querystring-parser@npm:3.224.0" +"@aws-sdk/querystring-parser@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/querystring-parser@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: d95aadf9c7af1b06ea3aeab3e734674ae744b6c674cbc4e9ce1f58177d930b56590c8a41b5e7eb1c9fa1c11afcad175813718d6de5b249cea36c8f0dfc81c041 + checksum: 13b23b4dff859d778a05f6874e318d95d6fbd79c62a06bb6f6dff8496ac2eaf86f60b17460c3595af854c2e316482c9fcc098655d475926039f7dff4a6f1ecf1 languageName: node linkType: hard -"@aws-sdk/service-error-classification@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/service-error-classification@npm:3.224.0" - checksum: e9942b92125170f0690ceb68819cfb72cfe0e6d040bc1452c14938eeb1b69c5c1b1bc4dadd85d5580447e51d21a92965203556e6893b02ad2a339f62a64090ad +"@aws-sdk/service-error-classification@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/service-error-classification@npm:3.226.0" + checksum: 4b3e9741ee58285b6bdec303b09090b638cc99ff09d2fac660d638da5fdf391ccfeb62f4af5065c975ed272040652c5c52557e599613af9577acfe3cf328ab04 languageName: node linkType: hard -"@aws-sdk/shared-ini-file-loader@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/shared-ini-file-loader@npm:3.224.0" +"@aws-sdk/shared-ini-file-loader@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/shared-ini-file-loader@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 3b03aa70a9396095152f0bb7e698bf00e572d9dc9ba166bb6eca73c0ad9694af9439beb40da2959711e4c44c6fdd12283a7c910e576bb4568429d1c207e9f823 + checksum: 7b43e4d70f33b1098c9582a20dd5cf11c0e2efe5e7f4f854ee1fdfe2f804a7171deeb76f167c46ef7365bb584e29c816d58b6d8f6c9ac9debb124cccf59ebf59 languageName: node linkType: hard -"@aws-sdk/signature-v4-multi-region@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/signature-v4-multi-region@npm:3.224.0" +"@aws-sdk/signature-v4-multi-region@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/signature-v4-multi-region@npm:3.226.0" dependencies: - "@aws-sdk/protocol-http": 3.224.0 - "@aws-sdk/signature-v4": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/protocol-http": 3.226.0 + "@aws-sdk/signature-v4": 3.226.0 + "@aws-sdk/types": 3.226.0 "@aws-sdk/util-arn-parser": 3.208.0 tslib: ^2.3.1 peerDependencies: @@ -1319,45 +1319,45 @@ __metadata: peerDependenciesMeta: "@aws-sdk/signature-v4-crt": optional: true - checksum: 28eed25747a1e52a1bae5fe3909c0e0aba243a4d76908746f9e3a6418b4fc1ac5f14db5108334969231d2086d739fea9f7bc3c5c79daa08761d95406dd02a5b3 + checksum: 173645989956907bbd83c4098e2e52f977272073713ac894ec7bf74f84ddb071c2142b2f5edfb7f7f939a60b8d8b5140bd114e6b028bb5216c341da7eb910ea2 languageName: node linkType: hard -"@aws-sdk/signature-v4@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/signature-v4@npm:3.224.0" +"@aws-sdk/signature-v4@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/signature-v4@npm:3.226.0" dependencies: "@aws-sdk/is-array-buffer": 3.201.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 "@aws-sdk/util-hex-encoding": 3.201.0 - "@aws-sdk/util-middleware": 3.224.0 + "@aws-sdk/util-middleware": 3.226.0 "@aws-sdk/util-uri-escape": 3.201.0 tslib: ^2.3.1 - checksum: d25d5058ca95e602dee993405e2b3766c866d5f1e1084dac3140f32e390e8dddb50aae716e415571eec73691ec1c3134f2377c4742c8310c11a4eaea1911a4cc + checksum: 8bc6ea1ad467d59f1ff6c5b814277e16d0d5b2d1455f21969b4575f572fdc013284e356e0f24436c89c8b99a51d0e384b01daa2bace1c3d69e28f35a23ef9802 languageName: node linkType: hard -"@aws-sdk/smithy-client@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/smithy-client@npm:3.224.0" +"@aws-sdk/smithy-client@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/smithy-client@npm:3.226.0" dependencies: - "@aws-sdk/middleware-stack": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/middleware-stack": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: fa42228e506b4114e97ee3de1b1c1980d65ada4248eb9637e5a1e4b132b7ce25adc7581628a3e4638de9344e9ab8994b8560c172349a3024626f599e430e125e + checksum: 7c77d26367b94286fd0eceb5036f3a931314ab24c01ae64f645ea1390b3dc94d443a751be2be504d36aeeef86c5412facdc7df9624a792d9e97e5d53531f850c languageName: node linkType: hard -"@aws-sdk/token-providers@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/token-providers@npm:3.224.0" +"@aws-sdk/token-providers@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/token-providers@npm:3.226.0" dependencies: - "@aws-sdk/client-sso-oidc": 3.224.0 - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/shared-ini-file-loader": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/client-sso-oidc": 3.226.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/shared-ini-file-loader": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 88f930b3f58cd3e31786b06083aef40fdc572139523745ab361935986552625f0ddff5fd08f18046d992a43931887968aab278f16bc7ea93c0b611f8bade24c2 + checksum: f9e26e7126f7df5ed94af129e75d2910c5784c3c37e79fdf940645bc07dee44e8cddaa487fac19bceab445c473be6864a49b9471f0b6df95607df32f7b786a99 languageName: node linkType: hard @@ -1368,21 +1368,23 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/types@npm:3.224.0, @aws-sdk/types@npm:^3.1.0, @aws-sdk/types@npm:^3.110.0, @aws-sdk/types@npm:^3.208.0": - version: 3.224.0 - resolution: "@aws-sdk/types@npm:3.224.0" - checksum: fa6682baf21dbd50b54e8ccde97ce26c69f18c60a6a7a8e4099526d89b46159d77534b1756db8d3e7ca408b9283b8e5685d329572c85ea9686a49f2e42b9c822 +"@aws-sdk/types@npm:3.226.0, @aws-sdk/types@npm:^3.1.0, @aws-sdk/types@npm:^3.110.0, @aws-sdk/types@npm:^3.208.0": + version: 3.226.0 + resolution: "@aws-sdk/types@npm:3.226.0" + dependencies: + tslib: ^2.3.1 + checksum: 0041a8c0924ec7ba4ff787ae329a80b3edb9ace43e38fe3656a1862ae5324427fb09836275c0962cbc80e3c34a6ee512a108a5c4e7997f29664e3e8930cffd80 languageName: node linkType: hard -"@aws-sdk/url-parser@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/url-parser@npm:3.224.0" +"@aws-sdk/url-parser@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/url-parser@npm:3.226.0" dependencies: - "@aws-sdk/querystring-parser": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/querystring-parser": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: ac7007c3ce03610ded32e7d77f211733434b08b28be0f94e1ee9e6e228463d95e69b4abe4f977e2f0c3836c3a4350a5041fa787b4363e822ec888c41b32da754 + checksum: cb8d4b13c0de7336728a7a60a246108c0ecd46e5e493cb1006f5ab58c5dbae0ac36a791536786b1facc90f58a8d6decca9f4fda2cba19667ba5a9f5f4fcadb63 languageName: node linkType: hard @@ -1442,39 +1444,39 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/util-defaults-mode-browser@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/util-defaults-mode-browser@npm:3.224.0" +"@aws-sdk/util-defaults-mode-browser@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/util-defaults-mode-browser@npm:3.226.0" dependencies: - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/types": 3.226.0 bowser: ^2.11.0 tslib: ^2.3.1 - checksum: 02a90d352a80bab1ddd4799dc74c4fb11d786e33088bf08d52ff9514cbad481480995bf4ad5353579eb8bc30e98e8bf059e5213a3be383db0008996188ff366a + checksum: 80a1383ef46c9289b7ef88ed1223e07f06bd3989517157199e325492d3da465d42bd9a975432b6d7c0e7e11f21aea02e5d977ddcc28c9f8b9f13e172fce0e657 languageName: node linkType: hard -"@aws-sdk/util-defaults-mode-node@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/util-defaults-mode-node@npm:3.224.0" +"@aws-sdk/util-defaults-mode-node@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/util-defaults-mode-node@npm:3.226.0" dependencies: - "@aws-sdk/config-resolver": 3.224.0 - "@aws-sdk/credential-provider-imds": 3.224.0 - "@aws-sdk/node-config-provider": 3.224.0 - "@aws-sdk/property-provider": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/config-resolver": 3.226.0 + "@aws-sdk/credential-provider-imds": 3.226.0 + "@aws-sdk/node-config-provider": 3.226.0 + "@aws-sdk/property-provider": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 26b6e76e75a3a658cdcfb1dcecddd28e732fd9dd69e288872d44631283322f70cd85a6da00bcc8d491ecf7c29a6eb2cddd06c8813a4844b7dac04467404ce7bf + checksum: cb34426f745aa3965fe25d5c8e38eb4d1c3f85f6f071995b15ecad48afcd0ca01a01b7f572c26dacd917134d9c676625492df8f6fb01d702ad6becb65bd34105 languageName: node linkType: hard -"@aws-sdk/util-endpoints@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/util-endpoints@npm:3.224.0" +"@aws-sdk/util-endpoints@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/util-endpoints@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: ffc030b7cd991364b4c89e0ed5b91a886c214bfbfbca7ef8a8aa8006e443a17d0c47a21b0f80b8a22878299bd31ae2153a8e4ee309372a13c678900e52949f00 + checksum: a05d38c4901801ff8c33d3469b8248ae07a2bb761e8b87f9eb62d7b977bd447fc8a1601c18692a035662818b223be97957d63d389b5393fdd65227cc25aa7903 languageName: node linkType: hard @@ -1496,38 +1498,38 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/util-middleware@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/util-middleware@npm:3.224.0" +"@aws-sdk/util-middleware@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/util-middleware@npm:3.226.0" dependencies: tslib: ^2.3.1 - checksum: 8e84a4ca626e5eeb087817685f28a064dc969225df1f295ac6152ab9613c8b6ef13c70ae181fe23b35dcdcdbfd1359f0d0ffb0dbf841c403359227f6fd44db6a + checksum: 51a4ba9a784943b723a2c57bbb889a4bc743f0f1219000ef73512e259069d63c3baa805aa375277c9d904da23301d0389bc0bd525b0b55274d7658fb62173e55 languageName: node linkType: hard -"@aws-sdk/util-stream-browser@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/util-stream-browser@npm:3.224.0" +"@aws-sdk/util-stream-browser@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/util-stream-browser@npm:3.226.0" dependencies: - "@aws-sdk/fetch-http-handler": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/fetch-http-handler": 3.226.0 + "@aws-sdk/types": 3.226.0 "@aws-sdk/util-base64": 3.208.0 "@aws-sdk/util-hex-encoding": 3.201.0 "@aws-sdk/util-utf8-browser": 3.188.0 tslib: ^2.3.1 - checksum: 904b2cbf307786e80900a35db85a807299a0a5cb87e1466069173d5f262d96955d6c0c2e4fed13099726ccb5b5f738a86ea1b8430bb41a8bbf0f45a3d4667802 + checksum: 7e232bf20733618b25f1dfee93d53a397cf22bdc7191e920b649837773fc6e551c1671f71581d6294d9e77c4b00dabb7baa182e85cedad470642e743a5b05a5c languageName: node linkType: hard -"@aws-sdk/util-stream-node@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/util-stream-node@npm:3.224.0" +"@aws-sdk/util-stream-node@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/util-stream-node@npm:3.226.0" dependencies: - "@aws-sdk/node-http-handler": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/node-http-handler": 3.226.0 + "@aws-sdk/types": 3.226.0 "@aws-sdk/util-buffer-from": 3.208.0 tslib: ^2.3.1 - checksum: e97fc82dd5071d1c4b84a644730505b43e8a039d8515913e3d1bd00c475eca6fdfb3550b281af939ebb14da79bef2f7f50e8f2459ff0b05732827b25c7b20b65 + checksum: 91ed91380582bb1989d7420d4939af240fcaf0805845a308ebb9917a1b5da3cae23ebcc9963fcd20c362467130951fc13666e181eb8d40da42f213eaebcada00 languageName: node linkType: hard @@ -1545,30 +1547,30 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/util-user-agent-browser@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/util-user-agent-browser@npm:3.224.0" +"@aws-sdk/util-user-agent-browser@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/util-user-agent-browser@npm:3.226.0" dependencies: - "@aws-sdk/types": 3.224.0 + "@aws-sdk/types": 3.226.0 bowser: ^2.11.0 tslib: ^2.3.1 - checksum: 49866dc6c000d60b4ab387de732458b41bb534bc9d51fbd28ac57c166bf37bbe88c8652121fefc23740de909aa550b98f4e9cb39f4e27c4b37999df7d5d5f9dc + checksum: b89b63e03636b98ccb811e713776c8ee5ce92a280edfd429e418dc69bb55b770b47d6118419be76207afe137f293fdb2e87605922714189df8ddbd4573185717 languageName: node linkType: hard -"@aws-sdk/util-user-agent-node@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/util-user-agent-node@npm:3.224.0" +"@aws-sdk/util-user-agent-node@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/util-user-agent-node@npm:3.226.0" dependencies: - "@aws-sdk/node-config-provider": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/node-config-provider": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 peerDependencies: aws-crt: ">=1.0.0" peerDependenciesMeta: aws-crt: optional: true - checksum: b9b240ee2776776e2d44dfc198424a23b26b3a2bb3cdd107fb9d3fb2179476f93f93399991e1c447fe13c98d793676876c50c8d70a47eb3c34e98dfa0318da95 + checksum: a3b620fa13e5e61bc874d777d451ec522f659f933b7d7f7642c10fbb12ce13dd068c2b5370fd6132cca5ddcd22243fb04b4e926a4dd8aedeb08270a9bb49ee74 languageName: node linkType: hard @@ -1591,14 +1593,14 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/util-waiter@npm:3.224.0": - version: 3.224.0 - resolution: "@aws-sdk/util-waiter@npm:3.224.0" +"@aws-sdk/util-waiter@npm:3.226.0": + version: 3.226.0 + resolution: "@aws-sdk/util-waiter@npm:3.226.0" dependencies: - "@aws-sdk/abort-controller": 3.224.0 - "@aws-sdk/types": 3.224.0 + "@aws-sdk/abort-controller": 3.226.0 + "@aws-sdk/types": 3.226.0 tslib: ^2.3.1 - checksum: 7360833ecad2d0b2b6add1e36f09900f98227243ba44312e3a1363bfb180e987b3a35de64aee41edc15f9dc2c6b85e83d8fb11fa8fc46ef855cb83c08542b42a + checksum: d182033937bd5ab024598a48ad28030267c1b9aa54175f24d0e57ea87975a0a07b5da7d0e5f43efb663e908a5df591dfa307bf9707a02d84eb49b089e9c5a5db languageName: node linkType: hard From 7911f7601069c09d5dd5b24dea9275336d00ca48 Mon Sep 17 00:00:00 2001 From: Marcin Kozlowski Date: Thu, 8 Dec 2022 21:40:03 +0100 Subject: [PATCH 111/437] Addtion of Betterscan plugin Signed-off-by: Marcin Kozlowski --- microsite/data/plugins/betterscan.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 microsite/data/plugins/betterscan.yaml diff --git a/microsite/data/plugins/betterscan.yaml b/microsite/data/plugins/betterscan.yaml new file mode 100644 index 0000000000..92d1eaa7b2 --- /dev/null +++ b/microsite/data/plugins/betterscan.yaml @@ -0,0 +1,10 @@ +--- +title: Betterscan +author: Marcin Kozlowski +authorUrl: https://betterscan.io +category: Security +description: View security scanned vulnerabilities in Code and Cloud scanned using Open Source and proprietary scanners directly in Backstage. +documentation: https://github.com/marcinguy/betterscan-ce +iconUrl: https://uploads-ssl.webflow.com/6339e3b81867539b5fe2498d/633a1643dcb06d3029867161_g4.svg +npmPackageName: '@marcinguy/backstage-plugin-betterscan' +addedDate: '2022-12-08' From dd721148b51f97130b3fe11bcca1fc3aed663468 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 8 Dec 2022 22:33:20 +0100 Subject: [PATCH 112/437] cli: fix coverage config warnings Signed-off-by: Patrik Oldsberg --- .changeset/fast-walls-explode.md | 5 +++++ packages/cli/config/jest.js | 22 ++++++++++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-) create mode 100644 .changeset/fast-walls-explode.md diff --git a/.changeset/fast-walls-explode.md b/.changeset/fast-walls-explode.md new file mode 100644 index 0000000000..ac4c4335bc --- /dev/null +++ b/.changeset/fast-walls-explode.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Updated Jest coverage configuration to only apply either in the root project or package configuration, depending on whether repo or package tests are run. diff --git a/packages/cli/config/jest.js b/packages/cli/config/jest.js index 148d574441..cbe0fda0de 100644 --- a/packages/cli/config/jest.js +++ b/packages/cli/config/jest.js @@ -65,7 +65,7 @@ function getRoleConfig(role) { } } -async function getProjectConfig(targetPath, displayName) { +async function getProjectConfig(targetPath, extraConfig) { const configJsPath = path.resolve(targetPath, 'jest.config.js'); const configTsPath = path.resolve(targetPath, 'jest.config.ts'); // If the package has it's own jest config, we use that instead. @@ -125,11 +125,8 @@ async function getProjectConfig(targetPath, displayName) { } const options = { - ...(displayName && { displayName }), + ...extraConfig, rootDir: path.resolve(targetPath, 'src'), - coverageDirectory: path.resolve(targetPath, 'coverage'), - coverageProvider: envOptions.nextTests ? 'babel' : 'v8', - collectCoverageFrom: ['**/*.{js,jsx,ts,tsx,mjs,cjs}', '!**/*.d.ts'], moduleNameMapper: { '\\.(css|less|scss|sss|styl)$': require.resolve('jest-css-modules'), }, @@ -237,15 +234,21 @@ async function getRootConfig() { const targetPackagePath = path.resolve(targetPath, 'package.json'); const exists = await fs.pathExists(targetPackagePath); + const coverageConfig = { + coverageDirectory: path.resolve(targetPath, 'coverage'), + coverageProvider: envOptions.nextTests ? 'babel' : 'v8', + collectCoverageFrom: ['**/*.{js,jsx,ts,tsx,mjs,cjs}', '!**/*.d.ts'], + }; + if (!exists) { - return getProjectConfig(targetPath); + return getProjectConfig(targetPath, coverageConfig); } // Check whether the current package is a workspace root or not const data = await fs.readJson(targetPackagePath); const workspacePatterns = data.workspaces && data.workspaces.packages; if (!workspacePatterns) { - return getProjectConfig(targetPath); + return getProjectConfig(targetPath, coverageConfig); } // If the target package is a workspace root, we find all packages in the @@ -269,7 +272,9 @@ async function getRootConfig() { testScript?.includes('backstage-cli test') || testScript?.includes('backstage-cli package test'); if (testScript && isSupportedTestScript) { - return await getProjectConfig(projectPath, packageData.name); + return await getProjectConfig(projectPath, { + displayName: packageData.name, + }); } return undefined; @@ -279,6 +284,7 @@ async function getRootConfig() { return { rootDir: targetPath, projects: configs, + ...coverageConfig, }; } From 07283a62c1a36d2658da584b090aee50fc5def54 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 8 Dec 2022 22:54:39 +0100 Subject: [PATCH 113/437] cli: make next jest config the default Signed-off-by: Patrik Oldsberg --- .github/workflows/ci.yml | 2 -- .github/workflows/deploy_packages.yml | 1 - .github/workflows/verify_windows.yml | 1 - packages/cli/config/jest.js | 20 ++++++++++---------- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7cc5874184..65c2c76242 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -202,7 +202,6 @@ jobs: if: ${{ steps.yarn-lock.outcome == 'success' }} run: yarn backstage-cli repo test --maxWorkers=2 --workerIdleMemoryLimit=1300M --since origin/master env: - BACKSTAGE_NEXT_TESTS: 1 BACKSTAGE_TEST_DISABLE_DOCKER: 1 BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }} BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }} @@ -214,7 +213,6 @@ jobs: yarn backstage-cli repo test --maxWorkers=2 --workerIdleMemoryLimit=800M --coverage bash <(curl -s https://codecov.io/bash) -N $(git rev-parse FETCH_HEAD) env: - BACKSTAGE_NEXT_TESTS: 1 BACKSTAGE_TEST_DISABLE_DOCKER: 1 BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }} BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }} diff --git a/.github/workflows/deploy_packages.yml b/.github/workflows/deploy_packages.yml index bebe24d0f6..63dc93ba2c 100644 --- a/.github/workflows/deploy_packages.yml +++ b/.github/workflows/deploy_packages.yml @@ -105,7 +105,6 @@ jobs: bash <(curl -s https://codecov.io/bash) -f packages/core-components/coverage/* -F core-components bash <(curl -s https://codecov.io/bash) -f packages/core-plugin-api/coverage/* -F core-plugin-api env: - BACKSTAGE_NEXT_TESTS: 1 BACKSTAGE_TEST_DISABLE_DOCKER: 1 BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres13.ports[5432] }} BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING: postgresql://postgres:postgres@localhost:${{ job.services.postgres9.ports[5432] }} diff --git a/.github/workflows/verify_windows.yml b/.github/workflows/verify_windows.yml index 75a0633cbe..916dd53366 100644 --- a/.github/workflows/verify_windows.yml +++ b/.github/workflows/verify_windows.yml @@ -48,7 +48,6 @@ jobs: - name: test run: yarn backstage-cli repo test --maxWorkers=2 --workerIdleMemoryLimit=1300M env: - BACKSTAGE_NEXT_TESTS: 1 BACKSTAGE_TEST_DISABLE_DOCKER: 1 # credit: https://github.com/appleboy/discord-action/issues/3#issuecomment-731426861 diff --git a/packages/cli/config/jest.js b/packages/cli/config/jest.js index cbe0fda0de..2c224f8161 100644 --- a/packages/cli/config/jest.js +++ b/packages/cli/config/jest.js @@ -21,11 +21,11 @@ const glob = require('util').promisify(require('glob')); const { version } = require('../package.json'); const envOptions = { - nextTests: Boolean(process.env.BACKSTAGE_NEXT_TESTS), + oldTests: Boolean(process.env.BACKSTAGE_OLD_TESTS), enableSourceMaps: Boolean(process.env.ENABLE_SOURCE_MAPS), }; -if (envOptions.nextTests) { +if (!envOptions.oldTests) { // Needed so that, at import-time, it can hook into Jest's internals. require('./jestCachingModuleLoader'); } @@ -135,7 +135,7 @@ async function getProjectConfig(targetPath, extraConfig) { '\\.(mjs|cjs|js)$': [ require.resolve('./jestSwcTransform'), { - sourceMaps: envOptions.enableSourceMaps || envOptions.nextTests, + sourceMaps: envOptions.enableSourceMaps || !envOptions.oldTests, jsc: { parser: { syntax: 'ecmascript', @@ -146,7 +146,7 @@ async function getProjectConfig(targetPath, extraConfig) { '\\.jsx$': [ require.resolve('./jestSwcTransform'), { - sourceMaps: envOptions.enableSourceMaps || envOptions.nextTests, + sourceMaps: envOptions.enableSourceMaps || !envOptions.oldTests, jsc: { parser: { syntax: 'ecmascript', @@ -163,7 +163,7 @@ async function getProjectConfig(targetPath, extraConfig) { '\\.ts$': [ require.resolve('./jestSwcTransform'), { - sourceMaps: envOptions.enableSourceMaps || envOptions.nextTests, + sourceMaps: envOptions.enableSourceMaps || !envOptions.oldTests, jsc: { parser: { syntax: 'typescript', @@ -174,7 +174,7 @@ async function getProjectConfig(targetPath, extraConfig) { '\\.tsx$': [ require.resolve('./jestSwcTransform'), { - sourceMaps: envOptions.enableSourceMaps || envOptions.nextTests, + sourceMaps: envOptions.enableSourceMaps || !envOptions.oldTests, jsc: { parser: { syntax: 'typescript', @@ -196,9 +196,9 @@ async function getProjectConfig(targetPath, extraConfig) { // A bit more opinionated testMatch: ['**/*.test.{js,jsx,ts,tsx,mjs,cjs}'], - runtime: envOptions.nextTests - ? require.resolve('./jestCachingModuleLoader') - : undefined, + runtime: envOptions.oldTests + ? undefined + : require.resolve('./jestCachingModuleLoader'), transformIgnorePatterns: [`/node_modules/(?:${transformIgnorePattern})/`], ...getRoleConfig(closestPkgJson?.backstage?.role), @@ -236,7 +236,7 @@ async function getRootConfig() { const coverageConfig = { coverageDirectory: path.resolve(targetPath, 'coverage'), - coverageProvider: envOptions.nextTests ? 'babel' : 'v8', + coverageProvider: envOptions.oldTests ? 'v8' : 'babel', collectCoverageFrom: ['**/*.{js,jsx,ts,tsx,mjs,cjs}', '!**/*.d.ts'], }; From 00196b700ca13c1a7528e9cef7826ffd1a9419cc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 8 Dec 2022 23:11:02 +0100 Subject: [PATCH 114/437] cli: remove redundant caching from jest module loader Signed-off-by: Patrik Oldsberg --- packages/cli/config/jest.js | 5 -- .../cli/config/jestCachingModuleLoader.js | 58 ------------------- 2 files changed, 63 deletions(-) diff --git a/packages/cli/config/jest.js b/packages/cli/config/jest.js index 2c224f8161..24385a6b99 100644 --- a/packages/cli/config/jest.js +++ b/packages/cli/config/jest.js @@ -25,11 +25,6 @@ const envOptions = { enableSourceMaps: Boolean(process.env.ENABLE_SOURCE_MAPS), }; -if (!envOptions.oldTests) { - // Needed so that, at import-time, it can hook into Jest's internals. - require('./jestCachingModuleLoader'); -} - const transformIgnorePattern = [ '@material-ui', 'ajv', diff --git a/packages/cli/config/jestCachingModuleLoader.js b/packages/cli/config/jestCachingModuleLoader.js index 7858a7cfb7..95c6212123 100644 --- a/packages/cli/config/jestCachingModuleLoader.js +++ b/packages/cli/config/jestCachingModuleLoader.js @@ -14,61 +14,11 @@ * limitations under the License. */ -const fs = require('fs'); const { default: JestRuntime } = require('jest-runtime'); -const fileTransformCache = new Map(); const scriptTransformCache = new Map(); -let runtimeGeneration = 0; -let isWatchMode; - module.exports = class CachingJestRuntime extends JestRuntime { - // Each Jest run creates a new runtime, including when rerunning tests in - // watch mode. This keeps track of whether we've switched runtime instance. - __runtimeGeneration = runtimeGeneration++; - - transformFile(filename, options) { - if (!isWatchMode) { - return super.transformFile(filename, options); - } - - const entry = fileTransformCache.get(filename); - if (entry) { - // Only check modification time if it's from a different runtime generation - if (entry.generation === this.__runtimeGeneration) { - return entry.code; - } - - // Keep track of the modification time of files so that we can properly - // reprocess them in watch mode. - const { mtimeMs } = fs.statSync(filename); - if (mtimeMs > entry.mtimeMs) { - const code = super.transformFile(filename, options); - fileTransformCache.set(filename, { - code, - mtimeMs, - generation: this.__runtimeGeneration, - }); - return code; - } - - fileTransformCache.set(filename, { - ...entry, - generation: this.__runtimeGeneration, - }); - return entry.code; - } - - const code = super.transformFile(filename, options); - fileTransformCache.set(filename, { - code, - mtimeMs: fs.statSync(filename).mtimeMs, - generation: this.__runtimeGeneration, - }); - return code; - } - // This may or may not be a good idea. Theoretically I don't know why this would impact // test correctness and flakiness, but it seems like it may introduce flakiness and strange failures. // It does seem to speed up test execution by a fair amount though. @@ -84,11 +34,3 @@ module.exports = class CachingJestRuntime extends JestRuntime { return script; } }; - -// Inject hook into createHasteMap, as it's the only way that we can -// determine (from our scope here) if we're in "watch mode" or not. -const originalCreateHasteMap = JestRuntime.createHasteMap; -JestRuntime.createHasteMap = (config, options = undefined) => { - isWatchMode = options && options.watch; - return originalCreateHasteMap(config, options); -}; From 736f893f72bde44d53db1e503eba2587c9ad1880 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 8 Dec 2022 23:22:53 +0100 Subject: [PATCH 115/437] changesets: added changeset for moving to next tests by default Signed-off-by: Patrik Oldsberg --- .changeset/neat-insects-share.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/neat-insects-share.md diff --git a/.changeset/neat-insects-share.md b/.changeset/neat-insects-share.md new file mode 100644 index 0000000000..1fadb9b9a1 --- /dev/null +++ b/.changeset/neat-insects-share.md @@ -0,0 +1,7 @@ +--- +'@backstage/cli': minor +--- + +The Jest configuration that was previously enabled with `BACKSTAGE_NEXT_TESTS` is now enabled by default. To revert to the old configuration you can now instead set `BACKSTAGE_OLD_TESTS`. + +This new configuration uses the `babel` coverage provider rather than `v8`. It used to be that `v8` worked better when using Sucrase for transpilation, but now that we have switched to SWC, `babel` seems to work better. In addition, the new configuration also enables source maps by default, as they no longer have a negative impact on code coverage accuracy, and it also enables a modified Jest runtime with additional caching of script objects. From 09e1520a90252b8b87affa1dd6257429d23ee43e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 Dec 2022 00:47:23 +0000 Subject: [PATCH 116/437] Update dependency cronstrue to v2.21.0 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 9c77b522c3..4b048b35af 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19227,11 +19227,11 @@ __metadata: linkType: hard "cronstrue@npm:^2.2.0": - version: 2.20.0 - resolution: "cronstrue@npm:2.20.0" + version: 2.21.0 + resolution: "cronstrue@npm:2.21.0" bin: cronstrue: bin/cli.js - checksum: 31b145cdeca5260fb120b072a9c69a12f722b895fbfea4792829efffd78a85c6e8eef1029a5a2563a12487ff6b06e717d1f117c5f156bf6a1ad7bb1b66d5a9fe + checksum: c77206339b22c30f0d0c584dad19614cd29d712a00afc3cd5d6ebab8b5e651883e72c04832b82b55c4dcf4f6a9daff6c6c7023364770f7dff82ada29616d40a5 languageName: node linkType: hard From 309f2daca4b3d9e12208b0497e2723950ded343c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 Dec 2022 00:48:31 +0000 Subject: [PATCH 117/437] Update dependency esbuild to ^0.16.0 Signed-off-by: Renovate Bot --- .changeset/renovate-1da1327.md | 6 + packages/cli/package.json | 2 +- plugins/scaffolder-backend/package.json | 2 +- yarn.lock | 237 +++++++++++++++++++++++- 4 files changed, 242 insertions(+), 5 deletions(-) create mode 100644 .changeset/renovate-1da1327.md diff --git a/.changeset/renovate-1da1327.md b/.changeset/renovate-1da1327.md new file mode 100644 index 0000000000..cb88ccf463 --- /dev/null +++ b/.changeset/renovate-1da1327.md @@ -0,0 +1,6 @@ +--- +'@backstage/cli': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Updated dependency `esbuild` to `^0.16.0`. diff --git a/packages/cli/package.json b/packages/cli/package.json index 0bd803d1db..87d5ca90ca 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -67,7 +67,7 @@ "commander": "^9.1.0", "css-loader": "^6.5.1", "diff": "^5.0.0", - "esbuild": "^0.15.0", + "esbuild": "^0.16.0", "esbuild-loader": "^2.18.0", "eslint": "^8.6.0", "eslint-config-prettier": "^8.3.0", diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index c56e25f1b4..d238036966 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -91,7 +91,7 @@ "@types/nunjucks": "^3.1.4", "@types/supertest": "^2.0.8", "@types/zen-observable": "^0.8.0", - "esbuild": "^0.15.0", + "esbuild": "^0.16.0", "jest-when": "^3.1.0", "mock-fs": "^5.1.0", "msw": "^0.49.0", diff --git a/yarn.lock b/yarn.lock index 9c77b522c3..4742dcfb3e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3657,7 +3657,7 @@ __metadata: css-loader: ^6.5.1 del: ^6.0.0 diff: ^5.0.0 - esbuild: ^0.15.0 + esbuild: ^0.16.0 esbuild-loader: ^2.18.0 eslint: ^8.6.0 eslint-config-prettier: ^8.3.0 @@ -7387,7 +7387,7 @@ __metadata: command-exists: ^1.2.9 compression: ^1.7.4 cors: ^2.8.5 - esbuild: ^0.15.0 + esbuild: ^0.16.0 express: ^4.17.1 express-promise-router: ^4.1.0 fs-extra: 10.1.0 @@ -9094,6 +9094,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/android-arm64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/android-arm64@npm:0.16.3" + conditions: os=android & cpu=arm64 + languageName: node + linkType: hard + "@esbuild/android-arm@npm:0.15.18": version: 0.15.18 resolution: "@esbuild/android-arm@npm:0.15.18" @@ -9101,6 +9108,69 @@ __metadata: languageName: node linkType: hard +"@esbuild/android-arm@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/android-arm@npm:0.16.3" + conditions: os=android & cpu=arm + languageName: node + linkType: hard + +"@esbuild/android-x64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/android-x64@npm:0.16.3" + conditions: os=android & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/darwin-arm64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/darwin-arm64@npm:0.16.3" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/darwin-x64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/darwin-x64@npm:0.16.3" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/freebsd-arm64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/freebsd-arm64@npm:0.16.3" + conditions: os=freebsd & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/freebsd-x64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/freebsd-x64@npm:0.16.3" + conditions: os=freebsd & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/linux-arm64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/linux-arm64@npm:0.16.3" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/linux-arm@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/linux-arm@npm:0.16.3" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + +"@esbuild/linux-ia32@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/linux-ia32@npm:0.16.3" + conditions: os=linux & cpu=ia32 + languageName: node + linkType: hard + "@esbuild/linux-loong64@npm:0.15.18": version: 0.15.18 resolution: "@esbuild/linux-loong64@npm:0.15.18" @@ -9108,6 +9178,90 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-loong64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/linux-loong64@npm:0.16.3" + conditions: os=linux & cpu=loong64 + languageName: node + linkType: hard + +"@esbuild/linux-mips64el@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/linux-mips64el@npm:0.16.3" + conditions: os=linux & cpu=mips64el + languageName: node + linkType: hard + +"@esbuild/linux-ppc64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/linux-ppc64@npm:0.16.3" + conditions: os=linux & cpu=ppc64 + languageName: node + linkType: hard + +"@esbuild/linux-riscv64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/linux-riscv64@npm:0.16.3" + conditions: os=linux & cpu=riscv64 + languageName: node + linkType: hard + +"@esbuild/linux-s390x@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/linux-s390x@npm:0.16.3" + conditions: os=linux & cpu=s390x + languageName: node + linkType: hard + +"@esbuild/linux-x64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/linux-x64@npm:0.16.3" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/netbsd-x64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/netbsd-x64@npm:0.16.3" + conditions: os=netbsd & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/openbsd-x64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/openbsd-x64@npm:0.16.3" + conditions: os=openbsd & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/sunos-x64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/sunos-x64@npm:0.16.3" + conditions: os=sunos & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/win32-arm64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/win32-arm64@npm:0.16.3" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/win32-ia32@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/win32-ia32@npm:0.16.3" + conditions: os=win32 & cpu=ia32 + languageName: node + linkType: hard + +"@esbuild/win32-x64@npm:0.16.3": + version: 0.16.3 + resolution: "@esbuild/win32-x64@npm:0.16.3" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + "@eslint/eslintrc@npm:^1.3.3": version: 1.3.3 resolution: "@eslint/eslintrc@npm:1.3.3" @@ -21192,7 +21346,7 @@ __metadata: languageName: node linkType: hard -"esbuild@npm:^0.15.0, esbuild@npm:^0.15.6": +"esbuild@npm:^0.15.6": version: 0.15.18 resolution: "esbuild@npm:0.15.18" dependencies: @@ -21269,6 +21423,83 @@ __metadata: languageName: node linkType: hard +"esbuild@npm:^0.16.0": + version: 0.16.3 + resolution: "esbuild@npm:0.16.3" + dependencies: + "@esbuild/android-arm": 0.16.3 + "@esbuild/android-arm64": 0.16.3 + "@esbuild/android-x64": 0.16.3 + "@esbuild/darwin-arm64": 0.16.3 + "@esbuild/darwin-x64": 0.16.3 + "@esbuild/freebsd-arm64": 0.16.3 + "@esbuild/freebsd-x64": 0.16.3 + "@esbuild/linux-arm": 0.16.3 + "@esbuild/linux-arm64": 0.16.3 + "@esbuild/linux-ia32": 0.16.3 + "@esbuild/linux-loong64": 0.16.3 + "@esbuild/linux-mips64el": 0.16.3 + "@esbuild/linux-ppc64": 0.16.3 + "@esbuild/linux-riscv64": 0.16.3 + "@esbuild/linux-s390x": 0.16.3 + "@esbuild/linux-x64": 0.16.3 + "@esbuild/netbsd-x64": 0.16.3 + "@esbuild/openbsd-x64": 0.16.3 + "@esbuild/sunos-x64": 0.16.3 + "@esbuild/win32-arm64": 0.16.3 + "@esbuild/win32-ia32": 0.16.3 + "@esbuild/win32-x64": 0.16.3 + dependenciesMeta: + "@esbuild/android-arm": + optional: true + "@esbuild/android-arm64": + optional: true + "@esbuild/android-x64": + optional: true + "@esbuild/darwin-arm64": + optional: true + "@esbuild/darwin-x64": + optional: true + "@esbuild/freebsd-arm64": + optional: true + "@esbuild/freebsd-x64": + optional: true + "@esbuild/linux-arm": + optional: true + "@esbuild/linux-arm64": + optional: true + "@esbuild/linux-ia32": + optional: true + "@esbuild/linux-loong64": + optional: true + "@esbuild/linux-mips64el": + optional: true + "@esbuild/linux-ppc64": + optional: true + "@esbuild/linux-riscv64": + optional: true + "@esbuild/linux-s390x": + optional: true + "@esbuild/linux-x64": + optional: true + "@esbuild/netbsd-x64": + optional: true + "@esbuild/openbsd-x64": + optional: true + "@esbuild/sunos-x64": + optional: true + "@esbuild/win32-arm64": + optional: true + "@esbuild/win32-ia32": + optional: true + "@esbuild/win32-x64": + optional: true + bin: + esbuild: bin/esbuild + checksum: c2986b0433c6048b917c185067ea42427413ef4136c45012e180e48fc24e6f01af9c94ca7e9bc6dd29ac529af45d26c9d4eb5b8639c9a79f68f337d24aeda2af + languageName: node + linkType: hard + "escalade@npm:^3.1.1": version: 3.1.1 resolution: "escalade@npm:3.1.1" From ba1897b354424be32f1329b3aff5f7e5a15b5cda Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 Dec 2022 02:04:54 +0000 Subject: [PATCH 118/437] Update dependency @types/jscodeshift to v0.11.6 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 4b048b35af..ef0f0deb4f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14315,12 +14315,12 @@ __metadata: linkType: hard "@types/jscodeshift@npm:^0.11.0": - version: 0.11.5 - resolution: "@types/jscodeshift@npm:0.11.5" + version: 0.11.6 + resolution: "@types/jscodeshift@npm:0.11.6" dependencies: ast-types: ^0.14.1 recast: ^0.20.3 - checksum: 5929f729477792a2c745289399ac0e2c0c46d4970031fa188073154262c6b0fcb03cf926d70a9fbcdc4c299df0e7fa1f0d6548e6bd1bb03c8245918c5b1a60de + checksum: 418d34488f74e711b37fcfce5129df3494d7fd30e852b3e80ff659ef0b1a4a83911e62883a10d7e1111316b569fa91a5a6a6d22fee86c2d69db829e25b534b27 languageName: node linkType: hard From 4cd5dd52985681a518d0eb364c5d1c8956017279 Mon Sep 17 00:00:00 2001 From: Jeff Tian Date: Fri, 9 Dec 2022 13:11:42 +0800 Subject: [PATCH 119/437] docs: fix broken links The latest working link for building docker should be https://backstage.io/docs/deployment/docker. Signed-off-by: Jeff Tian --- contrib/docs/tutorials/aws-deployment.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/docs/tutorials/aws-deployment.md b/contrib/docs/tutorials/aws-deployment.md index a767d292bf..6edbb73f9d 100644 --- a/contrib/docs/tutorials/aws-deployment.md +++ b/contrib/docs/tutorials/aws-deployment.md @@ -1,7 +1,7 @@ # Deploying Backstage on AWS using ECR and EKS Backstage documentation shows how to build a [Docker -image](https://backstage.io/docs/getting-started/deployment-docker); this +image](https://backstage.io/docs/deployment/docker); this tutorial shows how to deploy that Docker image to AWS using Elastic Container Registry (ECR) and Elastic Kubernetes Service (EKS). Amazon also supports deployments with Helm, covered in the [Helm @@ -36,7 +36,7 @@ Go to [AWS IAM console](https://console.aws.amazon.com/iam/home) and select ## Publish a Backstage build Follow the [Docker -image](https://backstage.io/docs/getting-started/deployment-docker) +image](https://backstage.io/docs/deployment/docker) documentation to build a new Backstage Docker image: ```shell From e0d8f93ff4de2223a9aebd8f6f8217798986f0a6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 Dec 2022 09:06:03 +0000 Subject: [PATCH 120/437] Update dependency @swc/core to v1.3.22 Signed-off-by: Renovate Bot --- storybook/yarn.lock | 86 ++++++++++++++++++++++----------------------- yarn.lock | 86 ++++++++++++++++++++++----------------------- 2 files changed, 86 insertions(+), 86 deletions(-) diff --git a/storybook/yarn.lock b/storybook/yarn.lock index ac821fc16c..033225c60d 100644 --- a/storybook/yarn.lock +++ b/storybook/yarn.lock @@ -2966,90 +2966,90 @@ __metadata: languageName: node linkType: hard -"@swc/core-darwin-arm64@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-darwin-arm64@npm:1.3.21" +"@swc/core-darwin-arm64@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-darwin-arm64@npm:1.3.22" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@swc/core-darwin-x64@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-darwin-x64@npm:1.3.21" +"@swc/core-darwin-x64@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-darwin-x64@npm:1.3.22" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@swc/core-linux-arm-gnueabihf@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-linux-arm-gnueabihf@npm:1.3.21" +"@swc/core-linux-arm-gnueabihf@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-linux-arm-gnueabihf@npm:1.3.22" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@swc/core-linux-arm64-gnu@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-linux-arm64-gnu@npm:1.3.21" +"@swc/core-linux-arm64-gnu@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-linux-arm64-gnu@npm:1.3.22" conditions: os=linux & cpu=arm64 & libc=glibc languageName: node linkType: hard -"@swc/core-linux-arm64-musl@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-linux-arm64-musl@npm:1.3.21" +"@swc/core-linux-arm64-musl@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-linux-arm64-musl@npm:1.3.22" conditions: os=linux & cpu=arm64 & libc=musl languageName: node linkType: hard -"@swc/core-linux-x64-gnu@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-linux-x64-gnu@npm:1.3.21" +"@swc/core-linux-x64-gnu@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-linux-x64-gnu@npm:1.3.22" conditions: os=linux & cpu=x64 & libc=glibc languageName: node linkType: hard -"@swc/core-linux-x64-musl@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-linux-x64-musl@npm:1.3.21" +"@swc/core-linux-x64-musl@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-linux-x64-musl@npm:1.3.22" conditions: os=linux & cpu=x64 & libc=musl languageName: node linkType: hard -"@swc/core-win32-arm64-msvc@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-win32-arm64-msvc@npm:1.3.21" +"@swc/core-win32-arm64-msvc@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-win32-arm64-msvc@npm:1.3.22" conditions: os=win32 & cpu=arm64 languageName: node linkType: hard -"@swc/core-win32-ia32-msvc@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-win32-ia32-msvc@npm:1.3.21" +"@swc/core-win32-ia32-msvc@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-win32-ia32-msvc@npm:1.3.22" conditions: os=win32 & cpu=ia32 languageName: node linkType: hard -"@swc/core-win32-x64-msvc@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-win32-x64-msvc@npm:1.3.21" +"@swc/core-win32-x64-msvc@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-win32-x64-msvc@npm:1.3.22" conditions: os=win32 & cpu=x64 languageName: node linkType: hard "@swc/core@npm:^1.3.9": - version: 1.3.21 - resolution: "@swc/core@npm:1.3.21" + version: 1.3.22 + resolution: "@swc/core@npm:1.3.22" dependencies: - "@swc/core-darwin-arm64": 1.3.21 - "@swc/core-darwin-x64": 1.3.21 - "@swc/core-linux-arm-gnueabihf": 1.3.21 - "@swc/core-linux-arm64-gnu": 1.3.21 - "@swc/core-linux-arm64-musl": 1.3.21 - "@swc/core-linux-x64-gnu": 1.3.21 - "@swc/core-linux-x64-musl": 1.3.21 - "@swc/core-win32-arm64-msvc": 1.3.21 - "@swc/core-win32-ia32-msvc": 1.3.21 - "@swc/core-win32-x64-msvc": 1.3.21 + "@swc/core-darwin-arm64": 1.3.22 + "@swc/core-darwin-x64": 1.3.22 + "@swc/core-linux-arm-gnueabihf": 1.3.22 + "@swc/core-linux-arm64-gnu": 1.3.22 + "@swc/core-linux-arm64-musl": 1.3.22 + "@swc/core-linux-x64-gnu": 1.3.22 + "@swc/core-linux-x64-musl": 1.3.22 + "@swc/core-win32-arm64-msvc": 1.3.22 + "@swc/core-win32-ia32-msvc": 1.3.22 + "@swc/core-win32-x64-msvc": 1.3.22 dependenciesMeta: "@swc/core-darwin-arm64": optional: true @@ -3073,7 +3073,7 @@ __metadata: optional: true bin: swcx: run_swcx.js - checksum: c66cd9320c595c68b87c8d90dc9a978099dd25a84c5e9795a8c7fec95fecdd8481da82076a828880a226ad2c0e57155c0a2b97768e99dee042a74182056bda46 + checksum: 5c6fa613502cbfae9985c7e97452649120e50e65695c8243111b422f6dddfebabdbc5ec94a9238e199e6a823a86906635e18e731f8ed45470bca8e4964d11659 languageName: node linkType: hard diff --git a/yarn.lock b/yarn.lock index ef0f0deb4f..83f9d536e2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13143,90 +13143,90 @@ __metadata: languageName: node linkType: hard -"@swc/core-darwin-arm64@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-darwin-arm64@npm:1.3.21" +"@swc/core-darwin-arm64@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-darwin-arm64@npm:1.3.22" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@swc/core-darwin-x64@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-darwin-x64@npm:1.3.21" +"@swc/core-darwin-x64@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-darwin-x64@npm:1.3.22" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@swc/core-linux-arm-gnueabihf@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-linux-arm-gnueabihf@npm:1.3.21" +"@swc/core-linux-arm-gnueabihf@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-linux-arm-gnueabihf@npm:1.3.22" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@swc/core-linux-arm64-gnu@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-linux-arm64-gnu@npm:1.3.21" +"@swc/core-linux-arm64-gnu@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-linux-arm64-gnu@npm:1.3.22" conditions: os=linux & cpu=arm64 & libc=glibc languageName: node linkType: hard -"@swc/core-linux-arm64-musl@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-linux-arm64-musl@npm:1.3.21" +"@swc/core-linux-arm64-musl@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-linux-arm64-musl@npm:1.3.22" conditions: os=linux & cpu=arm64 & libc=musl languageName: node linkType: hard -"@swc/core-linux-x64-gnu@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-linux-x64-gnu@npm:1.3.21" +"@swc/core-linux-x64-gnu@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-linux-x64-gnu@npm:1.3.22" conditions: os=linux & cpu=x64 & libc=glibc languageName: node linkType: hard -"@swc/core-linux-x64-musl@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-linux-x64-musl@npm:1.3.21" +"@swc/core-linux-x64-musl@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-linux-x64-musl@npm:1.3.22" conditions: os=linux & cpu=x64 & libc=musl languageName: node linkType: hard -"@swc/core-win32-arm64-msvc@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-win32-arm64-msvc@npm:1.3.21" +"@swc/core-win32-arm64-msvc@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-win32-arm64-msvc@npm:1.3.22" conditions: os=win32 & cpu=arm64 languageName: node linkType: hard -"@swc/core-win32-ia32-msvc@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-win32-ia32-msvc@npm:1.3.21" +"@swc/core-win32-ia32-msvc@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-win32-ia32-msvc@npm:1.3.22" conditions: os=win32 & cpu=ia32 languageName: node linkType: hard -"@swc/core-win32-x64-msvc@npm:1.3.21": - version: 1.3.21 - resolution: "@swc/core-win32-x64-msvc@npm:1.3.21" +"@swc/core-win32-x64-msvc@npm:1.3.22": + version: 1.3.22 + resolution: "@swc/core-win32-x64-msvc@npm:1.3.22" conditions: os=win32 & cpu=x64 languageName: node linkType: hard "@swc/core@npm:^1.3.9": - version: 1.3.21 - resolution: "@swc/core@npm:1.3.21" + version: 1.3.22 + resolution: "@swc/core@npm:1.3.22" dependencies: - "@swc/core-darwin-arm64": 1.3.21 - "@swc/core-darwin-x64": 1.3.21 - "@swc/core-linux-arm-gnueabihf": 1.3.21 - "@swc/core-linux-arm64-gnu": 1.3.21 - "@swc/core-linux-arm64-musl": 1.3.21 - "@swc/core-linux-x64-gnu": 1.3.21 - "@swc/core-linux-x64-musl": 1.3.21 - "@swc/core-win32-arm64-msvc": 1.3.21 - "@swc/core-win32-ia32-msvc": 1.3.21 - "@swc/core-win32-x64-msvc": 1.3.21 + "@swc/core-darwin-arm64": 1.3.22 + "@swc/core-darwin-x64": 1.3.22 + "@swc/core-linux-arm-gnueabihf": 1.3.22 + "@swc/core-linux-arm64-gnu": 1.3.22 + "@swc/core-linux-arm64-musl": 1.3.22 + "@swc/core-linux-x64-gnu": 1.3.22 + "@swc/core-linux-x64-musl": 1.3.22 + "@swc/core-win32-arm64-msvc": 1.3.22 + "@swc/core-win32-ia32-msvc": 1.3.22 + "@swc/core-win32-x64-msvc": 1.3.22 dependenciesMeta: "@swc/core-darwin-arm64": optional: true @@ -13250,7 +13250,7 @@ __metadata: optional: true bin: swcx: run_swcx.js - checksum: c66cd9320c595c68b87c8d90dc9a978099dd25a84c5e9795a8c7fec95fecdd8481da82076a828880a226ad2c0e57155c0a2b97768e99dee042a74182056bda46 + checksum: 5c6fa613502cbfae9985c7e97452649120e50e65695c8243111b422f6dddfebabdbc5ec94a9238e199e6a823a86906635e18e731f8ed45470bca8e4964d11659 languageName: node linkType: hard From 833872e55b758ea57d0212f66738b5009ba90973 Mon Sep 17 00:00:00 2001 From: djamaile Date: Fri, 9 Dec 2022 11:19:02 +0100 Subject: [PATCH 121/437] chore: use changeset feedback action Signed-off-by: djamaile --- .../workflows/automate_changeset_feedback.yml | 73 +------------------ 1 file changed, 4 insertions(+), 69 deletions(-) diff --git a/.github/workflows/automate_changeset_feedback.yml b/.github/workflows/automate_changeset_feedback.yml index 700eae7d0c..b6b5ea3d7e 100644 --- a/.github/workflows/automate_changeset_feedback.yml +++ b/.github/workflows/automate_changeset_feedback.yml @@ -21,75 +21,10 @@ jobs: if: github.repository == 'backstage/backstage' && github.event.pull_request.user.login != 'backstage-service' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: backstage/actions/changeset-feedback@v0.5.9 with: # Fetch the commit that's merged into the base rather than the target ref # This will let us diff only the contents of the PR, without fetching more history - ref: 'refs/pull/${{ github.event.pull_request.number }}/merge' - - - name: fetch base - run: git fetch --depth 1 origin ${{ github.base_ref }} - - # We avoid using the in-source script since this workflow has elevated permissions that we don't want to expose - - name: Generate Feedback - id: generate-feedback - run: | - rm -f generate.js - wget -O generate.js https://raw.githubusercontent.com/backstage/backstage/master/scripts/generate-changeset-feedback.js 1>&2 - node generate.js FETCH_HEAD > feedback.txt - - - name: Post Feedback - uses: actions/github-script@v6 - env: - ISSUE_NUMBER: ${{ github.event.pull_request.number }} - with: - script: | - const owner = "backstage"; - const repo = "backstage"; - const marker = ""; - const feedback = require('fs').readFileSync('feedback.txt', 'utf8'); - const issue_number = Number(process.env.ISSUE_NUMBER); - const body = feedback.trim() ? feedback + marker : undefined - - const existingComments = await github.paginate(github.rest.issues.listComments, { - owner, - repo, - issue_number, - }); - - const existingComment = existingComments.find((c) => - c.user.login === "github-actions[bot]" && - c.body.includes(marker) - ); - - if (existingComment) { - if (body) { - if (existingComment.body !== body) { - console.log(`updating existing comment in #${issue_number}`); - await github.rest.issues.updateComment({ - owner, - repo, - comment_id: existingComment.id, - body, - }); - } else { - console.log(`skipped update of identical comment in #${issue_number}`); - } - } else { - console.log(`removing comment from #${issue_number}`); - await github.rest.issues.deleteComment({ - owner, - repo, - comment_id: existingComment.id, - body, - }); - } - } else if (body) { - console.log(`creating comment for #${issue_number}`); - await github.rest.issues.createComment({ - owner, - repo, - issue_number, - body, - }); - } + diffRef: 'refs/pull/${{ github.event.pull_request.number }}/merge' + github-token: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }} + issue-number: ${{ steps.pr-number.outputs.pr-number }} From 429dfddb7262f1ff6a9299ed483e1ea354d5a1c2 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Fri, 9 Dec 2022 11:39:44 +0100 Subject: [PATCH 122/437] chore: update Bitbucket Cloud OpenAPI specification There were no changes impacting the codebase. Signed-off-by: Patrick Jungermann --- .../bitbucket-cloud.oas.json | 378 +++++++++++------- 1 file changed, 232 insertions(+), 146 deletions(-) diff --git a/plugins/bitbucket-cloud-common/bitbucket-cloud.oas.json b/plugins/bitbucket-cloud-common/bitbucket-cloud.oas.json index 843d009265..9de3c5f9b3 100644 --- a/plugins/bitbucket-cloud-common/bitbucket-cloud.oas.json +++ b/plugins/bitbucket-cloud-common/bitbucket-cloud.oas.json @@ -711,96 +711,6 @@ }, "parameters": [] }, - "/repositories/{workspace_slug}/{repo_slug}/override-settings": { - "get": { - "tags": ["Repositories"], - "description": "", - "summary": "Retrieve the inheritance state for repository settings", - "responses": { - "200": { - "description": "The repository setting inheritance state", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/repository_inheritance_state" - } - } - } - }, - "404": { - "description": "If no repository exists at this location", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/error" - } - } - } - } - }, - "security": [ - { - "oauth2": ["repository:admin"] - }, - { - "basic": [] - }, - { - "api_key": [] - } - ] - }, - "put": { - "tags": ["Repositories"], - "description": "", - "summary": "Set the inheritance state for repository settings\n ", - "responses": { - "204": { - "description": "The repository setting inheritance state was set and no content returned" - }, - "404": { - "description": "If no repository exists at this location", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/error" - } - } - } - } - }, - "security": [ - { - "oauth2": ["repository:admin"] - }, - { - "basic": [] - }, - { - "api_key": [] - } - ] - }, - "parameters": [ - { - "name": "repo_slug", - "in": "path", - "description": "This can either be the repository slug or the UUID of the repository,\nsurrounded by curly-braces, for example: `{repository UUID}`.\n", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "workspace_slug", - "in": "path", - "required": true, - "schema": { - "type": "string" - } - } - ] - }, "/repositories/{workspace}": { "get": { "tags": ["Repositories"], @@ -3641,7 +3551,7 @@ "/repositories/{workspace}/{repo_slug}/default-reviewers": { "get": { "tags": ["Pullrequests"], - "description": "Returns the repository's default reviewers.\n\nThese are the users that are automatically added as reviewers on every\nnew pull request that is created.", + "description": "Returns the repository's default reviewers.\n\nThese are the users that are automatically added as reviewers on every\nnew pull request that is created. To obtain the repository's default reviewers\nas well as the default reviewers inherited from the project, use the\n[effective-default-reveiwers](#api-repositories-workspace-repo-slug-effective-default-reviewers-get) endpoint.", "summary": "List default reviewers", "responses": { "200": { @@ -5050,6 +4960,66 @@ } ] }, + "/repositories/{workspace}/{repo_slug}/effective-default-reviewers": { + "get": { + "tags": ["Pullrequests"], + "description": "Returns the repository's effective default reviewers. This includes both default\nreviewers defined at the repository level as well as those inherited from its project.\n\nThese are the users that are automatically added as reviewers on every\nnew pull request that is created.\n\n```\n$ curl https://api.bitbucket.org/2.0/repositories/{workspace_slug}/{repo_slug}/effective-default-reviewers?page=1&pagelen=20\n{\n \"pagelen\": 20,\n \"values\": [\n {\n \"user\": {\n \"display_name\": \"Patrick Wolf\",\n \"uuid\": \"{9565301a-a3cf-4b5d-88f4-dd6af8078d7e}\"\n },\n \"reviewer_type\": \"project\",\n \"type\": \"default_reviewer\",\n },\n {\n \"user\": {\n \"display_name\": \"Davis Lee\",\n \"uuid\": \"{f0e0e8e9-66c1-4b85-a784-44a9eb9ef1a6}\"\n },\n \"reviewer_type\": \"repository\",\n \"type\": \"default_reviewer\",\n }\n ],\n \"page\": 1,\n \"size\": 2\n}\n```", + "summary": "List effective default reviewers", + "responses": { + "200": { + "description": "The paginated list of effective default reviewers", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/paginated_default_reviewer_and_type" + } + } + } + }, + "403": { + "description": "If the authenticated user does not have access to view the default reviewers", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/error" + } + } + } + } + }, + "security": [ + { + "oauth2": ["pullrequest"] + }, + { + "basic": [] + }, + { + "api_key": [] + } + ] + }, + "parameters": [ + { + "name": "repo_slug", + "in": "path", + "description": "This can either be the repository slug or the UUID of the repository,\nsurrounded by curly-braces, for example: `{repository UUID}`.\n", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "workspace", + "in": "path", + "description": "This can either be the workspace ID (slug) or the workspace UUID\nsurrounded by curly-braces, for example: `{workspace UUID}`.\n", + "required": true, + "schema": { + "type": "string" + } + } + ] + }, "/repositories/{workspace}/{repo_slug}/environments/": { "get": { "tags": ["Deployments"], @@ -7788,6 +7758,97 @@ } ] }, + "/repositories/{workspace}/{repo_slug}/override-settings": { + "get": { + "tags": ["Repositories"], + "description": "", + "summary": "Retrieve the inheritance state for repository settings", + "responses": { + "200": { + "description": "The repository setting inheritance state", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/repository_inheritance_state" + } + } + } + }, + "404": { + "description": "If no repository exists at this location", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/error" + } + } + } + } + }, + "security": [ + { + "oauth2": ["repository:admin"] + }, + { + "basic": [] + }, + { + "api_key": [] + } + ] + }, + "put": { + "tags": ["Repositories"], + "description": "", + "summary": "Set the inheritance state for repository settings\n ", + "responses": { + "204": { + "description": "The repository setting inheritance state was set and no content returned" + }, + "404": { + "description": "If no repository exists at this location", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/error" + } + } + } + } + }, + "security": [ + { + "oauth2": ["repository:admin"] + }, + { + "basic": [] + }, + { + "api_key": [] + } + ] + }, + "parameters": [ + { + "name": "repo_slug", + "in": "path", + "description": "This can either be the repository slug or the UUID of the repository,\nsurrounded by curly-braces, for example: `{repository UUID}`.\n", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "workspace", + "in": "path", + "description": "This can either be the workspace ID (slug) or the workspace UUID\nsurrounded by curly-braces, for example: `{workspace UUID}`.\n", + "required": true, + "schema": { + "type": "string" + } + } + ] + }, "/repositories/{workspace}/{repo_slug}/patch/{spec}": { "get": { "tags": ["Commits"], @@ -8075,6 +8136,16 @@ } } }, + "402": { + "description": "You have reached your plan's user limit and must upgrade before giving access to additional users.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/error" + } + } + } + }, "403": { "description": "The requesting user isn't an admin of the repository, or the authentication method was not via app password.", "content": { @@ -8141,7 +8212,7 @@ "/repositories/{workspace}/{repo_slug}/permissions-config/users": { "get": { "tags": ["Repositories"], - "description": "Returns a paginated list of explicit user permissions for the given repository.\n\nExample:\n\n```\n$ curl https://api.bitbucket.org/2.0/repositories/atlassian_tutorial/geordi/permissions-config/users\n\n{\n \"pagelen\": 10,\n \"values\": [\n {\n \"type\": \"repository_user_permission\",\n \"user\": {\n \"type\": \"user\",\n \"display_name\": \"Colin Cameron\",\n \"uuid\": \"{d301aafa-d676-4ee0-88be-962be7417567}\",\n \"account_id\": \"557058:ba8948b2-49da-43a9-9e8b-e7249b8e324a\"\n },\n \"permission\": \"admin\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian_tutorial/geordi/\n permissions-config/users/557058:ba8948b2-49da-43a9-9e8b-e7249b8e324a\"\n }\n }\n },\n {\n \"type\": \"repository_user_permission\",\n \"user\": {\n \"type\": \"user\",\n \"display_name\": \"Sean Conaty\",\n \"uuid\": \"{504c3b62-8120-4f0c-a7bc-87800b9d6f70}\",\n \"account_id\": \"557058:ba8948b2-49da-43a9-9e8b-e7249b8e324c\"\n },\n \"permission\": \"write\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0//repositories/atlassian_tutorial/geordi/\n permissions-config/users/557058:ba8948b2-49da-43a9-9e8b-e7249b8e324c\"\n }\n }\n }\n ],\n \"page\": 1,\n \"size\": 2\n}\n```", + "description": "Returns a paginated list of explicit user permissions for the given repository.\nThis endpoint does not support BBQL features.\n\nExample:\n\n```\n$ curl https://api.bitbucket.org/2.0/repositories/atlassian_tutorial/geordi/permissions-config/users\n\n{\n \"pagelen\": 10,\n \"values\": [\n {\n \"type\": \"repository_user_permission\",\n \"user\": {\n \"type\": \"user\",\n \"display_name\": \"Colin Cameron\",\n \"uuid\": \"{d301aafa-d676-4ee0-88be-962be7417567}\",\n \"account_id\": \"557058:ba8948b2-49da-43a9-9e8b-e7249b8e324a\"\n },\n \"permission\": \"admin\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian_tutorial/geordi/\n permissions-config/users/557058:ba8948b2-49da-43a9-9e8b-e7249b8e324a\"\n }\n }\n },\n {\n \"type\": \"repository_user_permission\",\n \"user\": {\n \"type\": \"user\",\n \"display_name\": \"Sean Conaty\",\n \"uuid\": \"{504c3b62-8120-4f0c-a7bc-87800b9d6f70}\",\n \"account_id\": \"557058:ba8948b2-49da-43a9-9e8b-e7249b8e324c\"\n },\n \"permission\": \"write\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0//repositories/atlassian_tutorial/geordi/\n permissions-config/users/557058:ba8948b2-49da-43a9-9e8b-e7249b8e324c\"\n }\n }\n }\n ],\n \"page\": 1,\n \"size\": 2\n}\n```", "summary": "List explicit user permissions for a repository", "responses": { "200": { @@ -8175,7 +8246,7 @@ } }, "404": { - "description": "No repository exists for the given repo slug and workspace.", + "description": "No repository exists for the given repository slug and workspace.", "content": { "application/json": { "schema": { @@ -8363,6 +8434,16 @@ } } }, + "402": { + "description": "You have reached your plan's user limit and must upgrade before giving access to additional users.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/error" + } + } + } + }, "403": { "description": "The requesting user isn't an admin of the repository, or the authentication method was not via app password.", "content": { @@ -8374,7 +8455,7 @@ } }, "404": { - "description": "One or more of the workspace, repo, and selected user doesn't exist for the given identifiers.", + "description": "One or more of the workspace, repository, and selected user doesn't exist for the given identifiers.", "content": { "application/json": { "schema": { @@ -10847,7 +10928,7 @@ "/repositories/{workspace}/{repo_slug}/pullrequests/activity": { "get": { "tags": ["Pullrequests"], - "description": "Returns a paginated list of the pull request's activity log.\n\nThis handler serves both a v20 and internal endpoint. The v20 endpoint\nreturns reviewer comments, updates, approvals and request changes. The internal\nendpoint includes those plus tasks and attachments.\n\nComments created on a file or a line of code have an inline property.\n\nComment example:\n```\n{\n \"pagelen\": 20,\n \"values\": [\n {\n \"comment\": {\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695/comments/118571088\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695/_/diff#comment-118571088\"\n }\n },\n \"deleted\": false,\n \"pullrequest\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n },\n \"content\": {\n \"raw\": \"inline with to a dn from lines\",\n \"markup\": \"markdown\",\n \"html\": \"

inline with to a dn from lines

\",\n \"type\": \"rendered\"\n },\n \"created_on\": \"2019-09-27T00:33:46.039178+00:00\",\n \"user\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n },\n \"created_on\": \"2019-09-27T00:33:46.039178+00:00\",\n \"user\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n },\n \"updated_on\": \"2019-09-27T00:33:46.055384+00:00\",\n \"inline\": {\n \"context_lines\": \"\",\n \"to\": null,\n \"path\": \"\",\n \"outdated\": false,\n \"from\": 211\n },\n \"type\": \"pullrequest_comment\",\n \"id\": 118571088\n },\n \"pull_request\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n }\n }\n ]\n}\n```\n\nUpdates include a state property of OPEN, MERGED, or DECLINED.\n\nUpdate example:\n```\n{\n \"pagelen\": 20,\n \"values\": [\n {\n \"update\": {\n \"description\": \"\",\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\",\n \"destination\": {\n \"commit\": {\n \"type\": \"commit\",\n \"hash\": \"6a2c16e4a152\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/commit/6a2c16e4a152\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/commits/6a2c16e4a152\"\n }\n }\n },\n \"branch\": {\n \"name\": \"master\"\n },\n \"repository\": {\n \"name\": \"Atlaskit-MK-2\",\n \"type\": \"repository\",\n \"full_name\": \"atlassian/atlaskit-mk-2\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2\"\n },\n \"avatar\": {\n \"href\": \"https://bytebucket.org/ravatar/%7B%7D?ts=js\"\n }\n },\n \"uuid\": \"{}\"\n }\n },\n \"reason\": \"\",\n \"source\": {\n \"commit\": {\n \"type\": \"commit\",\n \"hash\": \"728c8bad1813\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/commit/728c8bad1813\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/commits/728c8bad1813\"\n }\n }\n },\n \"branch\": {\n \"name\": \"username/NONE-add-onClick-prop-for-accessibility\"\n },\n \"repository\": {\n \"name\": \"Atlaskit-MK-2\",\n \"type\": \"repository\",\n \"full_name\": \"atlassian/atlaskit-mk-2\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2\"\n },\n \"avatar\": {\n \"href\": \"https://bytebucket.org/ravatar/%7B%7D?ts=js\"\n }\n },\n \"uuid\": \"{}\"\n }\n },\n \"state\": \"OPEN\",\n \"author\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n },\n \"date\": \"2019-05-10T06:48:25.305565+00:00\"\n },\n \"pull_request\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n }\n }\n ]\n}\n```\n\nApproval example:\n```\n{\n \"pagelen\": 20,\n \"values\": [\n {\n \"approval\": {\n \"date\": \"2019-09-27T00:37:19.849534+00:00\",\n \"pullrequest\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n },\n \"user\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n }\n },\n \"pull_request\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n }\n }\n ]\n}\n```", + "description": "Returns a paginated list of the pull request's activity log.\n\nThis handler serves both a v20 and internal endpoint. The v20 endpoint\nreturns reviewer comments, updates, approvals and request changes. The internal\nendpoint includes those plus tasks and attachments.\n\nComments created on a file or a line of code have an inline property.\n\nComment example:\n```\n{\n \"pagelen\": 20,\n \"values\": [\n {\n \"comment\": {\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695/comments/118571088\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695/_/diff#comment-118571088\"\n }\n },\n \"deleted\": false,\n \"pullrequest\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n },\n \"content\": {\n \"raw\": \"inline with to a dn from lines\",\n \"markup\": \"markdown\",\n \"html\": \"

inline with to a dn from lines

\",\n \"type\": \"rendered\"\n },\n \"created_on\": \"2019-09-27T00:33:46.039178+00:00\",\n \"user\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n },\n \"created_on\": \"2019-09-27T00:33:46.039178+00:00\",\n \"user\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n },\n \"updated_on\": \"2019-09-27T00:33:46.055384+00:00\",\n \"inline\": {\n \"context_lines\": \"\",\n \"to\": null,\n \"path\": \"\",\n \"outdated\": false,\n \"from\": 211\n },\n \"type\": \"pullrequest_comment\",\n \"id\": 118571088\n },\n \"pull_request\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n }\n }\n ]\n}\n```\n\nUpdates include a state property of OPEN, MERGED, or DECLINED.\n\nUpdate example:\n```\n{\n \"pagelen\": 20,\n \"values\": [\n {\n \"update\": {\n \"description\": \"\",\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\",\n \"destination\": {\n \"commit\": {\n \"type\": \"commit\",\n \"hash\": \"6a2c16e4a152\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/commit/6a2c16e4a152\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/commits/6a2c16e4a152\"\n }\n }\n },\n \"branch\": {\n \"name\": \"master\"\n },\n \"repository\": {\n \"name\": \"Atlaskit-MK-2\",\n \"type\": \"repository\",\n \"full_name\": \"atlassian/atlaskit-mk-2\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2\"\n },\n \"avatar\": {\n \"href\": \"https://bytebucket.org/ravatar/%7B%7D?ts=js\"\n }\n },\n \"uuid\": \"{}\"\n }\n },\n \"reason\": \"\",\n \"source\": {\n \"commit\": {\n \"type\": \"commit\",\n \"hash\": \"728c8bad1813\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/commit/728c8bad1813\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/commits/728c8bad1813\"\n }\n }\n },\n \"branch\": {\n \"name\": \"username/NONE-add-onClick-prop-for-accessibility\"\n },\n \"repository\": {\n \"name\": \"Atlaskit-MK-2\",\n \"type\": \"repository\",\n \"full_name\": \"atlassian/atlaskit-mk-2\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2\"\n },\n \"avatar\": {\n \"href\": \"https://bytebucket.org/ravatar/%7B%7D?ts=js\"\n }\n },\n \"uuid\": \"{}\"\n }\n },\n \"state\": \"OPEN\",\n \"author\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n },\n \"date\": \"2019-05-10T06:48:25.305565+00:00\"\n },\n \"pull_request\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n }\n }\n ]\n}\n```\n\nApproval example:\n```\n{\n \"pagelen\": 20,\n \"values\": [\n {\n \"approval\": {\n \"date\": \"2019-09-27T00:37:19.849534+00:00\",\n \"pullrequest\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n },\n \"user\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n }\n },\n \"pull_request\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n }\n }\n ]\n}\n```", "summary": "List a pull request activity log", "responses": { "200": { @@ -11043,7 +11124,7 @@ "/repositories/{workspace}/{repo_slug}/pullrequests/{pull_request_id}/activity": { "get": { "tags": ["Pullrequests"], - "description": "Returns a paginated list of the pull request's activity log.\n\nThis handler serves both a v20 and internal endpoint. The v20 endpoint\nreturns reviewer comments, updates, approvals and request changes. The internal\nendpoint includes those plus tasks and attachments.\n\nComments created on a file or a line of code have an inline property.\n\nComment example:\n```\n{\n \"pagelen\": 20,\n \"values\": [\n {\n \"comment\": {\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695/comments/118571088\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695/_/diff#comment-118571088\"\n }\n },\n \"deleted\": false,\n \"pullrequest\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n },\n \"content\": {\n \"raw\": \"inline with to a dn from lines\",\n \"markup\": \"markdown\",\n \"html\": \"

inline with to a dn from lines

\",\n \"type\": \"rendered\"\n },\n \"created_on\": \"2019-09-27T00:33:46.039178+00:00\",\n \"user\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n },\n \"created_on\": \"2019-09-27T00:33:46.039178+00:00\",\n \"user\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n },\n \"updated_on\": \"2019-09-27T00:33:46.055384+00:00\",\n \"inline\": {\n \"context_lines\": \"\",\n \"to\": null,\n \"path\": \"\",\n \"outdated\": false,\n \"from\": 211\n },\n \"type\": \"pullrequest_comment\",\n \"id\": 118571088\n },\n \"pull_request\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n }\n }\n ]\n}\n```\n\nUpdates include a state property of OPEN, MERGED, or DECLINED.\n\nUpdate example:\n```\n{\n \"pagelen\": 20,\n \"values\": [\n {\n \"update\": {\n \"description\": \"\",\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\",\n \"destination\": {\n \"commit\": {\n \"type\": \"commit\",\n \"hash\": \"6a2c16e4a152\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/commit/6a2c16e4a152\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/commits/6a2c16e4a152\"\n }\n }\n },\n \"branch\": {\n \"name\": \"master\"\n },\n \"repository\": {\n \"name\": \"Atlaskit-MK-2\",\n \"type\": \"repository\",\n \"full_name\": \"atlassian/atlaskit-mk-2\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2\"\n },\n \"avatar\": {\n \"href\": \"https://bytebucket.org/ravatar/%7B%7D?ts=js\"\n }\n },\n \"uuid\": \"{}\"\n }\n },\n \"reason\": \"\",\n \"source\": {\n \"commit\": {\n \"type\": \"commit\",\n \"hash\": \"728c8bad1813\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/commit/728c8bad1813\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/commits/728c8bad1813\"\n }\n }\n },\n \"branch\": {\n \"name\": \"username/NONE-add-onClick-prop-for-accessibility\"\n },\n \"repository\": {\n \"name\": \"Atlaskit-MK-2\",\n \"type\": \"repository\",\n \"full_name\": \"atlassian/atlaskit-mk-2\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2\"\n },\n \"avatar\": {\n \"href\": \"https://bytebucket.org/ravatar/%7B%7D?ts=js\"\n }\n },\n \"uuid\": \"{}\"\n }\n },\n \"state\": \"OPEN\",\n \"author\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n },\n \"date\": \"2019-05-10T06:48:25.305565+00:00\"\n },\n \"pull_request\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n }\n }\n ]\n}\n```\n\nApproval example:\n```\n{\n \"pagelen\": 20,\n \"values\": [\n {\n \"approval\": {\n \"date\": \"2019-09-27T00:37:19.849534+00:00\",\n \"pullrequest\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n },\n \"user\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n }\n },\n \"pull_request\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://bitbucket.org/!api/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n }\n }\n ]\n}\n```", + "description": "Returns a paginated list of the pull request's activity log.\n\nThis handler serves both a v20 and internal endpoint. The v20 endpoint\nreturns reviewer comments, updates, approvals and request changes. The internal\nendpoint includes those plus tasks and attachments.\n\nComments created on a file or a line of code have an inline property.\n\nComment example:\n```\n{\n \"pagelen\": 20,\n \"values\": [\n {\n \"comment\": {\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695/comments/118571088\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695/_/diff#comment-118571088\"\n }\n },\n \"deleted\": false,\n \"pullrequest\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n },\n \"content\": {\n \"raw\": \"inline with to a dn from lines\",\n \"markup\": \"markdown\",\n \"html\": \"

inline with to a dn from lines

\",\n \"type\": \"rendered\"\n },\n \"created_on\": \"2019-09-27T00:33:46.039178+00:00\",\n \"user\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n },\n \"created_on\": \"2019-09-27T00:33:46.039178+00:00\",\n \"user\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n },\n \"updated_on\": \"2019-09-27T00:33:46.055384+00:00\",\n \"inline\": {\n \"context_lines\": \"\",\n \"to\": null,\n \"path\": \"\",\n \"outdated\": false,\n \"from\": 211\n },\n \"type\": \"pullrequest_comment\",\n \"id\": 118571088\n },\n \"pull_request\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n }\n }\n ]\n}\n```\n\nUpdates include a state property of OPEN, MERGED, or DECLINED.\n\nUpdate example:\n```\n{\n \"pagelen\": 20,\n \"values\": [\n {\n \"update\": {\n \"description\": \"\",\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\",\n \"destination\": {\n \"commit\": {\n \"type\": \"commit\",\n \"hash\": \"6a2c16e4a152\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/commit/6a2c16e4a152\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/commits/6a2c16e4a152\"\n }\n }\n },\n \"branch\": {\n \"name\": \"master\"\n },\n \"repository\": {\n \"name\": \"Atlaskit-MK-2\",\n \"type\": \"repository\",\n \"full_name\": \"atlassian/atlaskit-mk-2\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2\"\n },\n \"avatar\": {\n \"href\": \"https://bytebucket.org/ravatar/%7B%7D?ts=js\"\n }\n },\n \"uuid\": \"{}\"\n }\n },\n \"reason\": \"\",\n \"source\": {\n \"commit\": {\n \"type\": \"commit\",\n \"hash\": \"728c8bad1813\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/commit/728c8bad1813\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/commits/728c8bad1813\"\n }\n }\n },\n \"branch\": {\n \"name\": \"username/NONE-add-onClick-prop-for-accessibility\"\n },\n \"repository\": {\n \"name\": \"Atlaskit-MK-2\",\n \"type\": \"repository\",\n \"full_name\": \"atlassian/atlaskit-mk-2\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2\"\n },\n \"avatar\": {\n \"href\": \"https://bytebucket.org/ravatar/%7B%7D?ts=js\"\n }\n },\n \"uuid\": \"{}\"\n }\n },\n \"state\": \"OPEN\",\n \"author\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n },\n \"date\": \"2019-05-10T06:48:25.305565+00:00\"\n },\n \"pull_request\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n }\n }\n ]\n}\n```\n\nApproval example:\n```\n{\n \"pagelen\": 20,\n \"values\": [\n {\n \"approval\": {\n \"date\": \"2019-09-27T00:37:19.849534+00:00\",\n \"pullrequest\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n },\n \"user\": {\n \"display_name\": \"Name Lastname\",\n \"uuid\": \"{}\",\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/users/%7B%7D\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/%7B%7D/\"\n },\n \"avatar\": {\n \"href\": \"https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/:/128\"\n }\n },\n \"type\": \"user\",\n \"nickname\": \"Name\",\n \"account_id\": \"\"\n }\n },\n \"pull_request\": {\n \"type\": \"pullrequest\",\n \"id\": 5695,\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/repositories/atlassian/atlaskit-mk-2/pullrequests/5695\"\n },\n \"html\": {\n \"href\": \"https://bitbucket.org/atlassian/atlaskit-mk-2/pull-requests/5695\"\n }\n },\n \"title\": \"username/NONE: small change from onFocus to onClick to handle tabbing through the page and not expand the editor unless a click event triggers it\"\n }\n }\n ]\n}\n```", "summary": "List a pull request activity log", "responses": { "200": { @@ -18035,6 +18116,7 @@ { "name": "project_key", "in": "path", + "description": "The project in question. This is the actual `key` assigned\nto the project.\n", "required": true, "schema": { "type": "string" @@ -18182,6 +18264,7 @@ { "name": "project_key", "in": "path", + "description": "The project in question. This is the actual `key` assigned\nto the project.\n", "required": true, "schema": { "type": "string" @@ -18201,7 +18284,7 @@ "/workspaces/{workspace}/projects/{project_key}/default-reviewers": { "get": { "tags": ["Projects"], - "description": "Return a list of all default reviewers for a project. This is a list of users that will be added as default\nreviewers to pull requests for any repository within the project.\n\nExample:\n```\n$ curl https://bitbucket.org/!api/2.0/.../projects/.../default-reviewers | jq .\n{\n \"pagelen\": 10,\n \"values\": [\n {\n \"user\": {\n \"display_name\": \"Davis Lee\",\n \"uuid\": \"{f0e0e8e9-66c1-4b85-a784-44a9eb9ef1a6}\"\n },\n \"reviewer_type\": \"project\",\n \"type\": \"default_reviewer\"\n },\n {\n \"user\": {\n \"display_name\": \"Jorge Rodriguez\",\n \"uuid\": \"{1aa43376-260d-4a0b-9660-f62672b9655d}\"\n },\n \"reviewer_type\": \"project\",\n \"type\": \"default_reviewer\"\n }\n ],\n \"page\": 1,\n \"size\": 2\n}\n```", + "description": "Return a list of all default reviewers for a project. This is a list of users that will be added as default\nreviewers to pull requests for any repository within the project.\n\nExample:\n```\n$ curl https://api.bitbucket.org/2.0/.../projects/.../default-reviewers | jq .\n{\n \"pagelen\": 10,\n \"values\": [\n {\n \"user\": {\n \"display_name\": \"Davis Lee\",\n \"uuid\": \"{f0e0e8e9-66c1-4b85-a784-44a9eb9ef1a6}\"\n },\n \"reviewer_type\": \"project\",\n \"type\": \"default_reviewer\"\n },\n {\n \"user\": {\n \"display_name\": \"Jorge Rodriguez\",\n \"uuid\": \"{1aa43376-260d-4a0b-9660-f62672b9655d}\"\n },\n \"reviewer_type\": \"project\",\n \"type\": \"default_reviewer\"\n }\n ],\n \"page\": 1,\n \"size\": 2\n}\n```", "summary": "List the default reviewers in a project", "responses": { "200": { @@ -18251,6 +18334,7 @@ { "name": "project_key", "in": "path", + "description": "The project in question. This is the actual `key` assigned\nto the project.\n", "required": true, "schema": { "type": "string" @@ -18270,7 +18354,7 @@ "/workspaces/{workspace}/projects/{project_key}/default-reviewers/{selected_user}": { "delete": { "tags": ["Projects"], - "description": "Removes a default reviewer from the project.\n\nExample:\n```\n$ curl https://bitbucket.org/!api/2.0/.../default-reviewers/%7Bf0e0e8e9-66c1-4b85-a784-44a9eb9ef1a6%7D\n\nHTTP/1.1 204\n```", + "description": "Removes a default reviewer from the project.\n\nExample:\n```\n$ curl https://api.bitbucket.org/2.0/.../default-reviewers/%7Bf0e0e8e9-66c1-4b85-a784-44a9eb9ef1a6%7D\n\nHTTP/1.1 204\n```", "summary": "Remove the specific user from the project's default reviewers", "responses": { "204": { @@ -18321,7 +18405,7 @@ }, "get": { "tags": ["Projects"], - "description": "Returns the specified default reviewer.\n\nExample:\n```\n$ curl https://bitbucket.org/!api/2.0/.../default-reviewers/%7Bf0e0e8e9-66c1-4b85-a784-44a9eb9ef1a6%7D\n{\n \"display_name\": \"Davis Lee\",\n \"type\": \"user\",\n \"uuid\": \"{f0e0e8e9-66c1-4b85-a784-44a9eb9ef1a6}\"\n}\n```", + "description": "Returns the specified default reviewer.\n\nExample:\n```\n$ curl https://api.bitbucket.org/2.0/.../default-reviewers/%7Bf0e0e8e9-66c1-4b85-a784-44a9eb9ef1a6%7D\n{\n \"display_name\": \"Davis Lee\",\n \"type\": \"user\",\n \"uuid\": \"{f0e0e8e9-66c1-4b85-a784-44a9eb9ef1a6}\"\n}\n```", "summary": "Get a default reviewer", "responses": { "200": { @@ -18379,7 +18463,7 @@ }, "put": { "tags": ["Projects"], - "description": "Adds the specified user to the project's list of default reviewers. The method is\nidempotent. Accepts an optional body containing the `uuid` of the user to be added.\n\nExample:\n```\n$ curl -XPUT https://bitbucket.org/!api/2.0/.../default-reviewers/%7Bf0e0e8e9-66c1-4b85-a784-44a9eb9ef1a6%7D\n-d { 'uuid': '{f0e0e8e9-66c1-4b85-a784-44a9eb9ef1a6}' }\n\nHTTP/1.1 204\n```", + "description": "Adds the specified user to the project's list of default reviewers. The method is\nidempotent. Accepts an optional body containing the `uuid` of the user to be added.\n\nExample:\n```\n$ curl -XPUT https://api.bitbucket.org/2.0/.../default-reviewers/%7Bf0e0e8e9-66c1-4b85-a784-44a9eb9ef1a6%7D\n-d { 'uuid': '{f0e0e8e9-66c1-4b85-a784-44a9eb9ef1a6}' }\n\nHTTP/1.1 204\n```", "summary": "Add the specific user as a default reviewer for the project", "responses": { "204": { @@ -18516,7 +18600,7 @@ }, "post": { "tags": ["Deployments"], - "description": "Create a new deploy key in a project.\n\nExample:\n```\n$ curl -XPOST \\\n-H \"Authorization \" \\\n-H \"Content-type: application/json\" \\\nhttps://api.bitbucket.org/!api/2.0/workspaces/jzeng/projects/JZ/deploy-keys/ -d \\\n'{\n \"key\": \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAK/b1cHHDr/TEV1JGQl+WjCwStKG6Bhrv0rFpEsYlyTBm1fzN0VOJJYn4ZOPCPJwqse6fGbXntEs+BbXiptR+++HycVgl65TMR0b5ul5AgwrVdZdT7qjCOCgaSV74/9xlHDK8oqgGnfA7ZoBBU+qpVyaloSjBdJfLtPY/xqj4yHnXKYzrtn/uFc4Kp9Tb7PUg9Io3qohSTGJGVHnsVblq/rToJG7L5xIo0OxK0SJSQ5vuId93ZuFZrCNMXj8JDHZeSEtjJzpRCBEXHxpOPhAcbm4MzULgkFHhAVgp4JbkrT99/wpvZ7r9AdkTg7HGqL3rlaDrEcWfL7Lu6TnhBdq5 mleu@C02W454JHTD8\",\n \"label\": \"mydeploykey\"\n}'\n\nOutput:\n{\n \"comment\": \"mleu@C02W454JHTD8\",\n \"last_used\": null,\n \"links\": {\n \"self\": {\n \"href\": \"https://jzeng.devbucket.org/!api/2.0/workspaces/testadfsa/projects/ASDF/deploy-keys/5/\"\n }\n },\n \"label\": \"myprojectkey\",\n \"project\": {\n ...\n },\n \"created_on\": \"2021-08-10T05:28:00.570859+00:00\",\n \"key\": \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAK/b1cHHDr/TEV1JGQl+WjCwStKG6Bhrv0rFpEsYlyTBm1fzN0VOJJYn4ZOPCPJwqse6fGbXntEs+BbXiptR+++HycVgl65TMR0b5ul5AgwrVdZdT7qjCOCgaSV74/9xlHDK8oqgGnfA7ZoBBU+qpVyaloSjBdJfLtPY/xqj4yHnXKYzrtn/uFc4Kp9Tb7PUg9Io3qohSTGJGVHnsVblq/rToJG7L5xIo0OxK0SJSQ5vuId93ZuFZrCNMXj8JDHZeSEtjJzpRCBEXHxpOPhAcbm4MzULgkFHhAVgp4JbkrT99/wpvZ7r9AdkTg7HGqL3rlaDrEcWfL7Lu6TnhBdq5\",\n \"type\": \"project_deploy_key\",\n \"id\": 5\n}\n```", + "description": "Create a new deploy key in a project.\n\nExample:\n```\n$ curl -XPOST \\\n-H \"Authorization \" \\\n-H \"Content-type: application/json\" \\\nhttps://api.bitbucket.org/2.0/workspaces/jzeng/projects/JZ/deploy-keys/ -d \\\n'{\n \"key\": \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAK/b1cHHDr/TEV1JGQl+WjCwStKG6Bhrv0rFpEsYlyTBm1fzN0VOJJYn4ZOPCPJwqse6fGbXntEs+BbXiptR+++HycVgl65TMR0b5ul5AgwrVdZdT7qjCOCgaSV74/9xlHDK8oqgGnfA7ZoBBU+qpVyaloSjBdJfLtPY/xqj4yHnXKYzrtn/uFc4Kp9Tb7PUg9Io3qohSTGJGVHnsVblq/rToJG7L5xIo0OxK0SJSQ5vuId93ZuFZrCNMXj8JDHZeSEtjJzpRCBEXHxpOPhAcbm4MzULgkFHhAVgp4JbkrT99/wpvZ7r9AdkTg7HGqL3rlaDrEcWfL7Lu6TnhBdq5 mleu@C02W454JHTD8\",\n \"label\": \"mydeploykey\"\n}'\n\nOutput:\n{\n \"comment\": \"mleu@C02W454JHTD8\",\n \"last_used\": null,\n \"links\": {\n \"self\": {\n \"href\": \"https://api.bitbucket.org/2.0/workspaces/testadfsa/projects/ASDF/deploy-keys/5/\"\n }\n },\n \"label\": \"myprojectkey\",\n \"project\": {\n ...\n },\n \"created_on\": \"2021-08-10T05:28:00.570859+00:00\",\n \"key\": \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAK/b1cHHDr/TEV1JGQl+WjCwStKG6Bhrv0rFpEsYlyTBm1fzN0VOJJYn4ZOPCPJwqse6fGbXntEs+BbXiptR+++HycVgl65TMR0b5ul5AgwrVdZdT7qjCOCgaSV74/9xlHDK8oqgGnfA7ZoBBU+qpVyaloSjBdJfLtPY/xqj4yHnXKYzrtn/uFc4Kp9Tb7PUg9Io3qohSTGJGVHnsVblq/rToJG7L5xIo0OxK0SJSQ5vuId93ZuFZrCNMXj8JDHZeSEtjJzpRCBEXHxpOPhAcbm4MzULgkFHhAVgp4JbkrT99/wpvZ7r9AdkTg7HGqL3rlaDrEcWfL7Lu6TnhBdq5\",\n \"type\": \"project_deploy_key\",\n \"id\": 5\n}\n```", "summary": "Create a project deploy key", "responses": { "200": { @@ -18576,6 +18660,7 @@ { "name": "project_key", "in": "path", + "description": "The project in question. This is the actual `key` assigned\nto the project.\n", "required": true, "schema": { "type": "string" @@ -18695,6 +18780,7 @@ { "name": "project_key", "in": "path", + "description": "The project in question. This is the actual `key` assigned\nto the project.\n", "required": true, "schema": { "type": "string" @@ -18894,7 +18980,7 @@ "description": "A workspace is where you create repositories, collaborate on\nyour code, and organize different streams of work in your Bitbucket\nCloud account. Workspaces replace the use of teams and users in API\ncalls.\n" } ], - "x-revision": "50c586d353cb", + "x-revision": "607fbe8bfcb2", "x-atlassian-narrative": { "documents": [ { @@ -18902,7 +18988,7 @@ "title": "Authentication methods", "description": "How to authenticate API actions", "icon": "data:image/svg+xml;base64,b'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxOTcuNjQ3MyAxODYuODEzOCI+CiAgPGRlZnM+CiAgICA8c3R5bGU+CiAgICAgIC5jbHMtMSB7CiAgICAgICAgaXNvbGF0aW9uOiBpc29sYXRlOwogICAgICB9CgogICAgICAuY2xzLTIgewogICAgICAgIGZpbGw6ICNkZTM1MGI7CiAgICAgIH0KCiAgICAgIC5jbHMtMyB7CiAgICAgICAgZmlsbDogI2ZmNTYzMDsKICAgICAgfQoKICAgICAgLmNscy00IHsKICAgICAgICBmaWxsOiAjZGZlMWU1OwogICAgICAgIG1peC1ibGVuZC1tb2RlOiBtdWx0aXBseTsKICAgICAgfQoKICAgICAgLmNscy01IHsKICAgICAgICBmaWxsOiAjZmFmYmZjOwogICAgICB9CgogICAgICAuY2xzLTYgewogICAgICAgIGZpbGw6ICNlYmVjZjA7CiAgICAgIH0KCiAgICAgIC5jbHMtNyB7CiAgICAgICAgZmlsbDogbm9uZTsKICAgICAgICBzdHJva2U6ICMwMDY1ZmY7CiAgICAgICAgc3Ryb2tlLW1pdGVybGltaXQ6IDEwOwogICAgICAgIHN0cm9rZS13aWR0aDogMnB4OwogICAgICB9CgogICAgICAuY2xzLTggewogICAgICAgIGZpbGw6ICM1ZTZjODQ7CiAgICAgIH0KCiAgICAgIC5jbHMtOSB7CiAgICAgICAgZmlsbDogIzI1Mzg1ODsKICAgICAgfQoKICAgICAgLmNscy0xMCB7CiAgICAgICAgZmlsbDogIzI2ODRmZjsKICAgICAgfQoKICAgICAgLmNscy0xMSB7CiAgICAgICAgZmlsbDogIzAwNjVmZjsKICAgICAgfQogICAgPC9zdHlsZT4KICA8L2RlZnM+CiAgPHRpdGxlPlNlY3VyaXR5IHdpdGggS2V5PC90aXRsZT4KICA8ZyBjbGFzcz0iY2xzLTEiPgogICAgPGcgaWQ9IkxheWVyXzIiIGRhdGEtbmFtZT0iTGF5ZXIgMiI+CiAgICAgIDxnIGlkPSJPYmplY3RzIj4KICAgICAgICA8cGF0aCBjbGFzcz0iY2xzLTIiIGQ9Ik00Mi4wNjcyLDBoLjYxMTRhOCw4LDAsMCwxLDgsOFYyMy4yMzM4YTAsMCwwLDAsMSwwLDBIMzQuMDY3MmEwLDAsMCwwLDEsMCwwVjhBOCw4LDAsMCwxLDQyLjA2NzIsMFoiLz4KICAgICAgICA8cGF0aCBjbGFzcz0iY2xzLTIiIGQ9Ik0xMDguMjIsMGguNjExNGE4LDgsMCwwLDEsOCw4VjIzLjIzMzhhMCwwLDAsMCwxLDAsMEgxMDAuMjJhMCwwLDAsMCwxLDAsMFY4QTgsOCwwLDAsMSwxMDguMjIsMFoiLz4KICAgICAgICA8cGF0aCBjbGFzcz0iY2xzLTIiIGQ9Ik0xNzQuMzcyMiwwaC42MTE0YTgsOCwwLDAsMSw4LDhWMjMuMjMzOGEwLDAsMCwwLDEsMCwwSDE2Ni4zNzIyYTAsMCwwLDAsMSwwLDBWOEE4LDgsMCwwLDEsMTc0LjM3MjIsMFoiLz4KICAgICAgICA8cmVjdCBjbGFzcz0iY2xzLTIiIHg9IjM0LjA2NzIiIHk9IjIzLjIzMzgiIHdpZHRoPSIxNjMuNTgiIGhlaWdodD0iMTYzLjU4Ii8+CiAgICAgICAgPHBhdGggY2xhc3M9ImNscy0yIiBkPSJNNDIuMDY3MiwwSDU5LjI5YTgsOCwwLDAsMSw4LDhWMjMuMjIyOGEwLDAsMCwwLDEsMCwwSDM0LjA2NzJhMCwwLDAsMCwxLDAsMFY4YTgsOCwwLDAsMSw4LThaIi8+CiAgICAgICAgPHBhdGggY2xhc3M9ImNscy0yIiBkPSJNMTA3LjI0NTgsMGgxNy4yMjI4YTgsOCwwLDAsMSw4LDhWMjMuMjIyOGEwLDAsMCwwLDEsMCwwSDk5LjI0NThhMCwwLDAsMCwxLDAsMFY4YTgsOCwwLDAsMSw4LThaIi8+CiAgICAgICAgPHBhdGggY2xhc3M9ImNscy0yIiBkPSJNMTcyLjQyNDQsMGgxNy4yMjI4YTgsOCwwLDAsMSw4LDhWMjMuMjIyOGEwLDAsMCwwLDEsMCwwSDE2NC40MjQ0YTAsMCwwLDAsMSwwLDBWOGE4LDgsMCwwLDEsOC04WiIvPgogICAgICAgIDxyZWN0IGNsYXNzPSJjbHMtMyIgeD0iMTcuNDU1OCIgeT0iMjMuMjMzOCIgd2lkdGg9IjE2My41OCIgaGVpZ2h0PSIxNjMuNTgiLz4KICAgICAgICA8cGF0aCBjbGFzcz0iY2xzLTMiIGQ9Ik0yNS40NTU4LDBINDIuNjc4NmE4LDgsMCwwLDEsOCw4VjIzLjIyMjhhMCwwLDAsMCwxLDAsMEgxNy40NTU4YTAsMCwwLDAsMSwwLDBWOEE4LDgsMCwwLDEsMjUuNDU1OCwwWiIvPgogICAgICAgIDxwYXRoIGNsYXNzPSJjbHMtMyIgZD0iTTkwLjYzNDQsMGgxNy4yMjI4YTgsOCwwLDAsMSw4LDhWMjMuMjIyOGEwLDAsMCwwLDEsMCwwSDgyLjYzNDRhMCwwLDAsMCwxLDAsMFY4QTgsOCwwLDAsMSw5MC42MzQ0LDBaIi8+CiAgICAgICAgPHBhdGggY2xhc3M9ImNscy0zIiBkPSJNMTU1LjgxMywwaDE3LjIyMjhhOCw4LDAsMCwxLDgsOFYyMy4yMjI4YTAsMCwwLDAsMSwwLDBIMTQ3LjgxM2EwLDAsMCwwLDEsMCwwVjhBOCw4LDAsMCwxLDE1NS44MTMsMFoiLz4KICAgICAgICA8cGF0aCBjbGFzcz0iY2xzLTMiIGQ9Ik0yNS40NTU4LDBINDIuNjc4NmE4LDgsMCwwLDEsOCw4VjIzLjIyMjhhMCwwLDAsMCwxLDAsMEgxNy40NTU4YTAsMCwwLDAsMSwwLDBWOEE4LDgsMCwwLDEsMjUuNDU1OCwwWiIvPgogICAgICAgIDxwYXRoIGNsYXNzPSJjbHMtMyIgZD0iTTkwLjYzNDQsMGgxNy4yMjI4YTgsOCwwLDAsMSw4LDhWMjMuMjIyOGEwLDAsMCwwLDEsMCwwSDgyLjYzNDRhMCwwLDAsMCwxLDAsMFY4QTgsOCwwLDAsMSw5MC42MzQ0LDBaIi8+CiAgICAgICAgPHBhdGggY2xhc3M9ImNscy0zIiBkPSJNMTU1LjgxMywwaDE3LjIyMjhhOCw4LDAsMCwxLDgsOFYyMy4yMjI4YTAsMCwwLDAsMSwwLDBIMTQ3LjgxM2EwLDAsMCwwLDEsMCwwVjhBOCw4LDAsMCwxLDE1NS44MTMsMFoiLz4KICAgICAgICA8cmVjdCBjbGFzcz0iY2xzLTIiIHg9IjM1Ljc1OTYiIHk9IjU2LjgwNjUiIHdpZHRoPSIzMy4yMjI4IiBoZWlnaHQ9IjE1LjYwMzgiLz4KICAgICAgICA8cmVjdCBjbGFzcz0iY2xzLTIiIHg9IjEzMS4yMDE2IiB5PSIxMzYuOTYxNSIgd2lkdGg9IjMzLjIyMjgiIGhlaWdodD0iMTUuNjAzOCIvPgogICAgICAgIDxwYXRoIGNsYXNzPSJjbHMtNCIgZD0iTTU3LjM3MDksNzEuNjAzNmg3MC43NWE5LDksMCwwLDEsOSw5djM1LjM3NDlhNDQuMzc0OCw0NC4zNzQ4LDAsMCwxLTQ0LjM3NDgsNDQuMzc0OGgwYTQ0LjM3NDgsNDQuMzc0OCwwLDAsMS00NC4zNzQ4LTQ0LjM3NDhWODAuNjAzNkE5LDksMCwwLDEsNTcuMzcwOSw3MS42MDM2WiIvPgogICAgICAgIDxwYXRoIGNsYXNzPSJjbHMtNSIgZD0iTTY2LjM3MSw2Ni42NjE3aDcwLjc1YTksOSwwLDAsMSw5LDl2MzUuMzc0OWE0NC4zNzQ4LDQ0LjM3NDgsMCwwLDEtNDQuMzc0OCw0NC4zNzQ4aDBBNDQuMzc0OCw0NC4zNzQ4LDAsMCwxLDU3LjM3MSwxMTEuMDM2NlY3NS42NjE3YTksOSwwLDAsMSw5LTlaIi8+CiAgICAgICAgPHBhdGggaWQ9Il9SZWN0YW5nbGVfIiBkYXRhLW5hbWU9IiZsdDtSZWN0YW5nbGUmZ3Q7IiBjbGFzcz0iY2xzLTYiIGQ9Ik02MS4zNzEsNjYuNjYxN2g3MC43NWE5LDksMCwwLDEsOSw5djM1LjM3NDlhNDQuMzc0OCw0NC4zNzQ4LDAsMCwxLTQ0LjM3NDgsNDQuMzc0OGgwQTQ0LjM3NDgsNDQuMzc0OCwwLDAsMSw1Mi4zNzEsMTExLjAzNjZWNzUuNjYxN0E5LDksMCwwLDEsNjEuMzcxLDY2LjY2MTdaIi8+CiAgICAgICAgPHBhdGggY2xhc3M9ImNscy03IiBkPSJNOTYuNzQ1OSwxNDcuNzQ0MWEzNi43NDg3LDM2Ljc0ODcsMCwwLDEtMzYuNzA3NC0zNi43MDc0Vjc4LjA1ODRhMy43MzMzLDMuNzMzMywwLDAsMSwzLjcyOS0zLjcyOWg2NS45NTYzYTMuNzMzMywzLjczMzMsMCwwLDEsMy43MjksMy43Mjl2MzIuOTc4NEEzNi43NDg2LDM2Ljc0ODYsMCwwLDEsOTYuNzQ1OSwxNDcuNzQ0MVoiLz4KICAgICAgICA8cGF0aCBjbGFzcz0iY2xzLTQiIGQ9Ik0xMDAuNjg5MywxNjMuMzE2N1YxMTEuMDk3M2EzLjk0NDMsMy45NDQzLDAsMCwwLTcuODg4NywwdjUyLjIyYTIyLjUyNTIsMjIuNTI1MiwwLDAsMC0xOC41NDc5LDIyLjE0YzAsLjQ1Ni4wMTc4LjkwNzguMDQ0NywxLjM1NzFIODIuMjFjLS4wNDE0LS40NDc0LS4wNjg4LS44OTktLjA2ODgtMS4zNTcxYTE0LjYyLDE0LjYyLDAsMCwxLDE0LjU5NzQtMTQuNjA0MWwuMDA2MS4wMDA2LjAwNjgtLjAwMDdBMTQuNjIxMSwxNC42MjExLDAsMCwxLDExMS4zNSwxODUuNDU2NmMwLC40NTgxLS4wMjczLjkxLS4wNjg4LDEuMzU3MWg3LjkxMjhjLjAyNjktLjQ0OTMuMDQ0Ny0uOTAxMS4wNDQ3LTEuMzU3MUEyMi41MjU5LDIyLjUyNTksMCwwLDAsMTAwLjY4OTMsMTYzLjMxNjdaIi8+CiAgICAgICAgPHJlY3QgY2xhc3M9ImNscy0yIiB4PSIxNy40NTU4IiB5PSIzNi40NzAyIiB3aWR0aD0iMzMuMjIyOCIgaGVpZ2h0PSIxNS42MDM4Ii8+CiAgICAgICAgPHJlY3QgY2xhc3M9ImNscy0yIiB4PSIxNy40NTU4IiB5PSIxNTguMTIxNyIgd2lkdGg9IjMzLjIyMjgiIGhlaWdodD0iMTUuNjAzOCIvPgogICAgICAgIDxyZWN0IGNsYXNzPSJjbHMtMiIgeD0iMTQ3LjgxMyIgeT0iMzYuNDcwMiIgd2lkdGg9IjMzLjIyMjgiIGhlaWdodD0iMTUuNjAzOCIvPgogICAgICAgIDxyZWN0IGNsYXNzPSJjbHMtMiIgeD0iMTUwLjA2NDMiIHk9IjE1Ny41NTEzIiB3aWR0aD0iMzMuMjIyOCIgaGVpZ2h0PSIxNS42MDM4Ii8+CiAgICAgICAgPHBhdGggaWQ9Il9QYXRoXyIgZGF0YS1uYW1lPSImbHQ7UGF0aCZndDsiIGNsYXNzPSJjbHMtOCIgZD0iTTEwNy41MjU0LDEwMS4wMDI3YTExLjc3OTQsMTEuNzc5NCwwLDEsMC0xOS44Niw4LjU1NDhBNC4wNDE3LDQuMDQxNywwLDAsMSw4OC44NSwxMTMuNjJsLTIuMTA0LDcuMjY4MWEzLDMsMCwwLDAsMi44ODE3LDMuODM0MmgxMi4yMzcxYTMsMywwLDAsMCwyLjg4MTctMy44MzQybC0yLjA5NTktNy4yNGE0LjA3NDMsNC4wNzQzLDAsMCwxLDEuMTgwOC00LjA5NDVBMTEuNzE3MiwxMS43MTcyLDAsMCwwLDEwNy41MjU0LDEwMS4wMDI3WiIvPgogICAgICAgIDxwYXRoIGNsYXNzPSJjbHMtOSIgZD0iTTEwNC43NDYxLDEyMC44ODc3bC0yLjA5NTktNy4yNGE0LjA3NDQsNC4wNzQ0LDAsMCwxLDEuMTgwOC00LjA5NDUsMTEuNzYyOSwxMS43NjI5LDAsMCwwLTUuMDYtMTkuOTMxMywxMS45MSwxMS45MSwwLDAsMC04Ljc5OCwxMC45OTQ5LDExLjcxODUsMTEuNzE4NSwwLDAsMCwzLjY5MjksOC45NDFBNC4wNDE2LDQuMDQxNiwwLDAsMSw5NC44NSwxMTMuNjJsLTMuMjE0LDExLjEwMjNoMTAuMjI4OEEzLDMsMCwwLDAsMTA0Ljc0NjEsMTIwLjg4NzdaIi8+CiAgICAgICAgPHBhdGggY2xhc3M9ImNscy0xMCIgZD0iTTgxLjc5NzUsMTAwLjMxYTMuOTQzOSwzLjk0MzksMCwwLDAtMy45NDQzLTMuOTQ0M0g0MS4wNDE3YTMuOTQ0MywzLjk0NDMsMCwwLDAsMCw3Ljg4ODdINzcuODUzMkEzLjk0MzksMy45NDM5LDAsMCwwLDgxLjc5NzUsMTAwLjMxWiIvPgogICAgICAgIDxwYXRoIGlkPSJfUGF0aF8yIiBkYXRhLW5hbWU9IiZsdDtQYXRoJmd0OyIgY2xhc3M9ImNscy0xMSIgZD0iTTQxLjA0MTYsMTA0LjI1MzlIOTYuODUzMmEzLjk0NDMsMy45NDQzLDAsMCwwLDAtNy44ODg3SDQxLjA0MTZhMy45NDQzLDMuOTQ0MywwLDAsMCwwLDcuODg4N1oiLz4KICAgICAgICA8cGF0aCBjbGFzcz0iY2xzLTEwIiBkPSJNODEuNzk3NSwxMDAuMzFhMy45NDM5LDMuOTQzOSwwLDAsMC0zLjk0NDMtMy45NDQzSDQxLjA0MTdhMy45NDQzLDMuOTQ0MywwLDAsMCwwLDcuODg4N0g3Ny44NTMyQTMuOTQzOSwzLjk0MzksMCwwLDAsODEuNzk3NSwxMDAuMzFaIi8+CiAgICAgICAgPHBhdGggY2xhc3M9ImNscy0xMCIgZD0iTTIyLjQ5MzIsMTIyLjgwMjlBMjIuNDkyOSwyMi40OTI5LDAsMSwxLDQ0Ljk4NTgsMTAwLjMxLDIyLjUxODUsMjIuNTE4NSwwLDAsMSwyMi40OTMyLDEyMi44MDI5Wm0wLTM3LjA5NzJBMTQuNjA0MiwxNC42MDQyLDAsMSwwLDM3LjA5NzIsMTAwLjMxLDE0LjYyMDcsMTQuNjIwNywwLDAsMCwyMi40OTMyLDg1LjcwNTdaIi8+CiAgICAgIDwvZz4KICAgIDwvZz4KICA8L2c+Cjwvc3ZnPgo='", - "body": "\nThe purpose of this section is to describe how to authenticate when making API calls using the Bitbucket REST API.\n\n-----\n\n* [Oauth 2](#oauth-2)\n * [Making requests](#making-requests)\n * [Repository cloning](#repository-cloning)\n * [Refresh tokens](#refresh-tokens)\n* [Scopes](#scopes)\n* [Basic auth](#basic-auth)\n* [App passwords](#app-passwords)\n\n---\n\n### OAuth 2.0\n\nOur OAuth 2 implementation is merged in with our existing OAuth 1 in\nsuch a way that existing OAuth 1 consumers automatically become\nvalid OAuth 2 clients. The only thing you need to do is edit your\nexisting consumer and configure a callback URL.\n\nOnce that is in place, you'll have the following 2 URLs:\n\n https://bitbucket.org/site/oauth2/authorize\n https://bitbucket.org/site/oauth2/access_token\n\nFor obtaining access/bearer tokens, we support three of RFC-6749's grant\nflows, plus a custom Bitbucket flow for exchanging JWT tokens for access tokens.\nNote that Resource Owner Password Credentials Grant (4.3) is no longer supported.\n\n\n#### 1. Authorization Code Grant (4.1)\n\nThe full-blown 3-LO flow. Request authorization from the end user by\nsending their browser to:\n\n https://bitbucket.org/site/oauth2/authorize?client_id={client_id}&response_type=code\n\nThe callback includes the `?code={}` query parameter that you can swap\nfor an access token:\n\n $ curl -X POST -u \"client_id:secret\" \\\n https://bitbucket.org/site/oauth2/access_token \\\n -d grant_type=authorization_code -d code={code}\n\n\n#### 2. Implicit Grant (4.2)\n\nThis flow is useful for browser-based add-ons that operate without server-side backends.\n\nRequest the end user for authorization by directing the browser to:\n\n https://bitbucket.org/site/oauth2/authorize?client_id={client_id}&response_type=token\n\nThat will redirect to your preconfigured callback URL with a fragment\ncontaining the access token\n(`#access_token={token}&token_type=bearer`) where your page's js can\npull it out of the URL.\n\n\n#### 3. Client Credentials Grant (4.4)\n\nSomewhat like our existing \"2-LO\" flow for OAuth 1. Obtain an access\ntoken that represents not an end user, but the owner of the\nclient/consumer:\n\n $ curl -X POST -u \"client_id:secret\" \\\n https://bitbucket.org/site/oauth2/access_token \\\n -d grant_type=client_credentials\n\n\n#### 4. Bitbucket Cloud JWT Grant (urn:bitbucket:oauth2:jwt)\n\nIf your Atlassian Connect add-on uses JWT authentication, you can swap a\nJWT for an OAuth access token. The resulting access token represents the\naccount for which the add-on is installed.\n\nMake sure you send the JWT token in the Authorization request header\nusing the \"JWT\" scheme (case sensitive). Note that this custom scheme\nmakes this different from HTTP Basic Auth (and so you cannot use \"curl\n-u\").\n\n $ curl -X POST -H \"Authorization: JWT {jwt_token}\" \\\n https://bitbucket.org/site/oauth2/access_token \\\n -d grant_type=urn:bitbucket:oauth2:jwt\n\n\n#### Making Requests\n\nOnce you have an access token, as per RFC-6750, you can use it in a request in any of\nthe following ways (in decreasing order of desirability):\n\n1. Send it in a request header: `Authorization: Bearer {access_token}`\n2. Include it in a (application/x-www-form-urlencoded) POST body as `access_token={access_token}`\n3. Put it in the query string of a non-POST: `?access_token={access_token}`\n\n\n#### Repository Cloning\n\nSince add-ons will not be able to upload their own SSH keys to clone\nwith, access tokens can be used as Basic HTTP Auth credentials to\nclone securely over HTTPS. This is much like GitHub, yet slightly\ndifferent:\n\n $ git clone https://x-token-auth:{access_token}@bitbucket.org/user/repo.git\n\nThe literal string `x-token-auth` as a substitute for username is\nrequired (note the difference with GitHub where the actual token is in\nthe username field).\n\n\n#### Refresh Tokens\n\nOur access tokens expire in one hour. When this happens you'll get 401\nresponses.\n\nMost access tokens grant responses (Implicit and JWT excluded). Therefore, you should include a\nrefresh token that can then be used to generate a new access token,\nwithout the need for end user participation:\n\n $ curl -X POST -u \"client_id:secret\" \\\n https://bitbucket.org/site/oauth2/access_token \\\n -d grant_type=refresh_token -d refresh_token={refresh_token}\n\n\n### Scopes\n\nBitbucket's API applies a number of privilege scopes to endpoints. In order to access an endpoint, a request will need to have the necessary scopes.\n\nScopes are declared in the descriptor as a list of strings, with each string being the name of a unique scope.\n\nA descriptor lacking the `scopes` element is implicitly assumed to require all scopes and as a result, Bitbucket will require end users authorizing/installing the add-on\nto explicitly accept all scopes.\n\nOur best practice suggests you add the scopes your add-on needs, but no more than it needs.\n\nInvalid scope strings will cause the descriptor to be rejected and the installation to fail.\n\nFollowing is the set of all currently available scopes.\n\n#### repository\n\nGives the add-on read access to all the repositories the authorizing user has access to.\nNote that this scope does not give access to a repository's pull requests.\n\n* access to the repo's source code\n* clone over https\n* access the the file browsing API\n* download zip archives of the repo's contents\n* the ability to view and use the issue tracker on any repo (created issues, comment, vote, etc)\n* the ability to view and use the wiki on any repo (create/edit pages)\n\n#### repository:write\n\nGives the add-on write (not admin) access to all the repositories the authorizing user has access to. No distinction is made between public or private repos. This scope implies `repository`, which does not need to be requested separately.\nThis scope alone does not give access to the pull requests API.\n\n* push access over https\n* fork repos\n\n#### repository:admin\n\nGives the add-on admin access to all the repositories the authorizing user has access to. No distinction is made between public or private repos. This scope does not imply `repository` or `repository:write`. It gives access to the admin features of a repo only, not direct access to its contents. Of course it can be (mis)used to grant read access to another user account who can then clone the repo, but repos that need to read of write source code would also request explicit read or write.\nThis scope comes with access to the following functionality:\n\n* view and manipulate committer mappings\n* list and edit deploy keys\n* ability to delete the repo\n* view and edit repo permissions\n* view and edit branch permissions\n* import and export the issue tracker\n* enable and disable the issue tracker\n* list and edit issue tracker version, milestones and components\n* enable and disable the wiki\n* list and edit default reviewers\n* list and edit repo links (Jira/Bamboo/Custom)\n* list and edit the repository web hooks\n* initiate a repo ownership transfer\n\n#### snippet\n\nGives the add-on read access to all the snippets the authorizing user has access to.\nNo distinction is made between public and private snippets (public snippets are accessible without any form of authentication).\n\n* view any snippet\n* create snippet comments\n\n#### snippet:write\n\nGives the add-on write access to all the snippets the authorizing user can edit.\nNo distinction is made between public and private snippets (public snippets are accessible without any form of authentication).\nThis implies the Snippet Read scope which does not need to be requested separately.\n\n* edit snippets\n* delete snippets\n\n#### issue\n\nAbility to interact with issue trackers the way non-repo members can.\nThis scope does not imply any other scopes and does not give implicit access to the repository the issue is attached to.\n\n* view, list and search issues\n* create new issues\n* comment on issues\n* watch issues\n* vote for issues\n\n#### issue:write\n\nThis implies `issue`, but adds the ability to transition and delete issues.\nThis scope does not imply any other scopes and does not give implicit access to the repository the issue is attached to.\n\n* transition issues\n* delete issues\n\n#### wiki\n\nGives access to wikis. No distinction is made between read and write as wikis are always editable by anyone.\nThis scope does not imply any other scopes and does not give implicit access to the repository the wiki is attached to.\n\n* view wikis\n* create pages\n* edit pages\n* push to wikis\n* clone wikis\n\n#### pullrequest\n\nGives the add-on read access to pull requests.\nThis scope implies `repository`, giving read access to the pull request's destination repository.\n\n* see and list pull requests\n* create and resolve tasks\n* comment on pull requests\n\n#### pullrequest:write\n\nImplies `pullrequest` but adds the ability to create, merge and decline pull requests.\nThis scope implies `repository:write`, giving write access to the pull request's destination repository. This is necessary to facilitate merging.\n\n* merge pull requests\n* decline pull requests\n* create pull requests\n* approve pull requests\n\n#### project\n\nGives the app `repository` scope permissions for every repository under every project that the authorizing user has read access to.\n\n#### project:write\n\nThis scope is deprecated, and has been made obsolete by `project:admin`. Please see the deprecation notice [here](/cloud/bitbucket/deprecation-notice-project-write-scope).\n\n#### project:admin\n\nGives the app admin access to all the projects the authorizing user has access to. No distinction is made between public or private projects. This scope does not imply `project`, or `repository:write` on any repositories under the project. It gives access to the admin features of a project only, not direct access to its repositories' contents.\n\n* ability to create the project\n* ability to update the project\n* ability to delete the project\n\n#### email\n\nAbility to see the user's primary email address. This should make it easier to use Bitbucket Cloud as a login provider to add-ons or external applications.\n\n#### account\n\nAbility to see all the user's account information. Note that this does not include any ability to mutate any of the data.\n\n* see all email addresses\n* language\n* location\n* website\n* full name\n* SSH keys\n* user groups\n\n#### account:write\n\nAbility to change properties on the user's account.\n\n* delete the authorizing user's account\n* manage the user's groups\n* manupilate a user's email addresses\n* change username, display name and avatar\n\n#### webhook\n\nGives access to webhooks. This scope is required for any webhook\nrelated operation.\n\nThis scope gives read access to existing webhook subscriptions on all\nresources you can access, without needing further scopes. This means that\na client can list all existing webhook subscriptions on repository\n`foo/bar` (assuming the principal user has access to this repo). The\nadditional `repository` scope is not required for this.\n\nLikewise, existing webhook subscriptions for a repo's issue tracker can be\nretrieved without holding the `issue` scope. All that is required is the\n`webhook` scope.\n\nHowever, to create a webhook for `issue:created`, the client will need to\nhave both the `webhook` as well as `issue` scope.\n\n* list webhook subscriptions on any accessible repository, user, team, or snippet\n* create/update/delete webhook subscriptions\n\n#### pipeline\n\nGives read-only access to pipelines, steps, deployment environments and variables.\n\n#### pipeline:write\n\nGives write access to pipelines. This scope allows a user to:\n* Stop pipelines\n* Rerun failed pipelines\n* Resume halted pipelines\n* Trigger manual pipelines.\n\nThis scope is not needed to trigger a build via a push. The act to doing push will trigger the build. The token doing the push only needs repository:write scope.\n\nThis does not give write access to create variables.\n\n#### pipeline:variable\n\nGives write access to create variables in pipelines at the various levels:\n* Workspace\n* Repository\n* Deployment\n\n#### runner\n\nGives read-only access to pipelines runners setup against a workspace or repository.\n\n#### runner:write\n\nGives write access to create/edit/disable/delete pipelines runners setup against a workspace or repository.\n\n### Basic auth\n\nBasic HTTP Authentication as per [RFC-2617](https://tools.ietf.org/html/rfc2617) (Digest not supported). Note that Basic Auth is available only with username and [app password](https://bitbucket.org/account/settings/app-passwords/) as credentials.\n\n### App passwords\n\nApp passwords allow users to make API calls to their Bitbucket account through apps such as Sourcetree.\n\nSome important points about app passwords:\n\n* You cannot view an app password or adjust permissions after you create the app password. Because app passwords are encrypted on our database and cannot be viewed by anyone. They are essentially designed to be disposable. If you need to change the scopes or lost the password just create a new one.\n* You cannot use them to log into your Bitbucket account.\n* You cannot use app passwords to manage team actions.\n\n App passwords are tied to an individual account's credentials and should not be shared. If you're sharing your app password you're essentially giving direct, authenticated, access to everything that password has been scoped to do with the Bitbucket API's.\n\n* You can use them for API call authentication, even if you don't have two-step verification enabled.\n* You can set permission scopes (specific access rights) for each app password.\n\n#### Create an app password\n\nTo create an app password:\n\n1. Select **Avatar > Bitbucket settings**.\n2. [Click **App passwords** in the Access management section.](https://bitbucket.org/account/settings/app-passwords/)\n3. Click **Create app password**.\n4. Give the app password a name related to the application that will use the password.\n5. Select the specific access and permissions you want this application password to have.\n6. Copy the generated password and either record or paste it into the application you want to give access. The password is only displayed this one time.\n\nThat's all there is to creating an app password. See your applications documentation for how to apply the app password for a specific application." + "body": "\nThe purpose of this section is to describe how to authenticate when making API calls using the Bitbucket REST API.\n\n-----\n\n* [Oauth 2](#oauth-2)\n * [Making requests](#making-requests)\n * [Repository cloning](#repository-cloning)\n * [Refresh tokens](#refresh-tokens)\n* [Scopes](#scopes)\n* [Basic auth](#basic-auth)\n* [Repository Access Tokens](#repository-access-tokens)\n* [App passwords](#app-passwords)\n\n---\n\n### OAuth 2.0\n\nOur OAuth 2 implementation is merged in with our existing OAuth 1 in\nsuch a way that existing OAuth 1 consumers automatically become\nvalid OAuth 2 clients. The only thing you need to do is edit your\nexisting consumer and configure a callback URL.\n\nOnce that is in place, you'll have the following 2 URLs:\n\n https://bitbucket.org/site/oauth2/authorize\n https://bitbucket.org/site/oauth2/access_token\n\nFor obtaining access/bearer tokens, we support three of RFC-6749's grant\nflows, plus a custom Bitbucket flow for exchanging JWT tokens for access tokens.\nNote that Resource Owner Password Credentials Grant (4.3) is no longer supported.\n\n\n#### 1. Authorization Code Grant (4.1)\n\nThe full-blown 3-LO flow. Request authorization from the end user by\nsending their browser to:\n\n https://bitbucket.org/site/oauth2/authorize?client_id={client_id}&response_type=code\n\nThe callback includes the `?code={}` query parameter that you can swap\nfor an access token:\n\n $ curl -X POST -u \"client_id:secret\" \\\n https://bitbucket.org/site/oauth2/access_token \\\n -d grant_type=authorization_code -d code={code}\n\n\n#### 2. Implicit Grant (4.2)\n\nThis flow is useful for browser-based add-ons that operate without server-side backends.\n\nRequest the end user for authorization by directing the browser to:\n\n https://bitbucket.org/site/oauth2/authorize?client_id={client_id}&response_type=token\n\nThat will redirect to your preconfigured callback URL with a fragment\ncontaining the access token\n(`#access_token={token}&token_type=bearer`) where your page's js can\npull it out of the URL.\n\n\n#### 3. Client Credentials Grant (4.4)\n\nSomewhat like our existing \"2-LO\" flow for OAuth 1. Obtain an access\ntoken that represents not an end user, but the owner of the\nclient/consumer:\n\n $ curl -X POST -u \"client_id:secret\" \\\n https://bitbucket.org/site/oauth2/access_token \\\n -d grant_type=client_credentials\n\n\n#### 4. Bitbucket Cloud JWT Grant (urn:bitbucket:oauth2:jwt)\n\nIf your Atlassian Connect add-on uses JWT authentication, you can swap a\nJWT for an OAuth access token. The resulting access token represents the\naccount for which the add-on is installed.\n\nMake sure you send the JWT token in the Authorization request header\nusing the \"JWT\" scheme (case sensitive). Note that this custom scheme\nmakes this different from HTTP Basic Auth (and so you cannot use \"curl\n-u\").\n\n $ curl -X POST -H \"Authorization: JWT {jwt_token}\" \\\n https://bitbucket.org/site/oauth2/access_token \\\n -d grant_type=urn:bitbucket:oauth2:jwt\n\n\n#### Making Requests\n\nOnce you have an access token, as per RFC-6750, you can use it in a request in any of\nthe following ways (in decreasing order of desirability):\n\n1. Send it in a request header: `Authorization: Bearer {access_token}`\n2. Include it in a (application/x-www-form-urlencoded) POST body as `access_token={access_token}`\n3. Put it in the query string of a non-POST: `?access_token={access_token}`\n\n\n#### Repository Cloning\n\nSince add-ons will not be able to upload their own SSH keys to clone\nwith, access tokens can be used as Basic HTTP Auth credentials to\nclone securely over HTTPS. This is much like GitHub, yet slightly\ndifferent:\n\n $ git clone https://x-token-auth:{access_token}@bitbucket.org/user/repo.git\n\nThe literal string `x-token-auth` as a substitute for username is\nrequired (note the difference with GitHub where the actual token is in\nthe username field).\n\n\n#### Refresh Tokens\n\nOur access tokens expire in one hour. When this happens you'll get 401\nresponses.\n\nMost access tokens grant responses (Implicit and JWT excluded). Therefore, you should include a\nrefresh token that can then be used to generate a new access token,\nwithout the need for end user participation:\n\n $ curl -X POST -u \"client_id:secret\" \\\n https://bitbucket.org/site/oauth2/access_token \\\n -d grant_type=refresh_token -d refresh_token={refresh_token}\n\n\n### Scopes\n\nBitbucket's API applies a number of privilege scopes to endpoints. In order to access an endpoint, a request will need to have the necessary scopes.\n\nScopes are declared in the descriptor as a list of strings, with each string being the name of a unique scope.\n\nA descriptor lacking the `scopes` element is implicitly assumed to require all scopes and as a result, Bitbucket will require end users authorizing/installing the add-on\nto explicitly accept all scopes.\n\nOur best practice suggests you add the scopes your add-on needs, but no more than it needs.\n\nInvalid scope strings will cause the descriptor to be rejected and the installation to fail.\n\nThe available scopes are:\n\n- [project](#project)\n- [project:write](#project-write)\n- [project:admin](#project-admin)\n- [repository](#repository)\n- [repository:write](#repository-write)\n- [repository:admin](#repository-admin)\n- [repository:delete](#repository-delete)\n- [pullrequest](#pullrequest)\n- [pullrequest:write](#pullrequest-write)\n- [issue](#issue)\n- [issue:write](#issue-write)\n- [wiki](#wiki)\n- [webhook](#webhook)\n- [snippet](#snippet)\n- [snippet:write](#snippet-write)\n- [email](#email)\n- [account](#account)\n- [account:write](#account-write)\n- [pipeline](#pipeline)\n- [pipeline:write](#pipeline-write)\n- [pipeline:variable](#pipeline-variable)\n- [runner](#runner)\n- [runner:write](#runner-write)\n\n#### project\n\nProvides the [`repository`](#repository) scope permission for every repository under a project or projects.\n\n#### project:write\n\nThis scope is deprecated, and has been made obsolete by `project:admin`. Please see the deprecation notice [here](/cloud/bitbucket/deprecation-notice-project-write-scope).\n\n#### project:admin\n\nProvides admin access to a project or projects. No distinction is made between public and private projects. This scope doesn't implicitly grant the [`project`](#project) scope or the [`repository:write`](#repository-write) scope on any repositories under the project. It gives access to the admin features of a project only, not direct access to its repositories' contents.\n\n* ability to create the project\n* ability to update the project\n* ability to delete the project\n\n#### repository\n\nProvides read access to a repository or repositories.\nNote that this scope does not give access to a repository's pull requests.\n\n* access to the repo's source code\n* clone over HTTPS\n* access the file browsing API\n* download zip archives of the repo's contents\n* the ability to view and use the issue tracker on any repo (created issues, comment, vote, etc)\n* the ability to view and use the wiki on any repo (create/edit pages)\n\n#### repository:write\n\nProvides write (not admin) access to a repository or repositories. No distinction is made between public and private repositories. This scope implicitly grants the [`repository`](#repository) scope, which does not need to be requested separately.\nThis scope alone does not give access to the pull requests API.\n\n* push access over HTTPS\n* fork repos\n\n#### repository:admin\n\nProvides admin access to a repository or repositories. No distinction is made between public and private repositories. This scope doesn't implicitly grant the [`repository`](#repository) or the [`repository:write`](#repository-write) scopes. It gives access to the admin features of a repo only, not direct access to its contents. This scope can be used or misused to grant read access to other users, who can then clone the repo, but users that need to read and write source code would also request explicit read or write.\nThis scope comes with access to the following functionality:\n\n* view and manipulate committer mappings\n* list and edit deploy keys\n* ability to delete the repo\n* view and edit repo permissions\n* view and edit branch permissions\n* import and export the issue tracker\n* enable and disable the issue tracker\n* list and edit issue tracker version, milestones and components\n* enable and disable the wiki\n* list and edit default reviewers\n* list and edit repo links (Jira/Bamboo/Custom)\n* list and edit the repository webhooks\n* initiate a repo ownership transfer\n\n#### repository:delete\n\nProvides access to delete a repository or repositories.\n\n#### pullrequest\n\nProvides read access to pull requests.\nThis scope implies the [`repository`](#repository) scope, giving read access to the pull request's destination repository.\n\n* see and list pull requests\n* create and resolve tasks\n* comment on pull requests\n\n#### pullrequest:write\n\nImplicitly grants the [`pullrequest`](#pullrequest) scope and adds the ability to create, merge and decline pull requests.\nThis scope also implicitly grants the [`repository:write`](#repository-write) scope, giving write access to the pull request's destination repository. This is necessary to allow merging.\n\n* merge pull requests\n* decline pull requests\n* create pull requests\n* approve pull requests\n\n#### issue\n\nAbility to interact with issue trackers the way non-repo members can.\nThis scope doesn't implicitly grant any other scopes and doesn't give implicit access to the repository.\n\n* view, list and search issues\n* create new issues\n* comment on issues\n* watch issues\n* vote for issues\n\n#### issue:write\n\nThis scope implicitly grants the [`issue`](#issue) scope and adds the ability to transition and delete issues.\nThis scope doesn't implicitly grant any other scopes and doesn't give implicit access to the repository.\n\n* transition issues\n* delete issues\n\n#### wiki\n\nProvides access to wikis. This scope provides both read and write access (wikis are always editable by anyone with access to them).\nThis scope doesn't implicitly grant any other scopes and doesn't give implicit access to the repository.\n\n* view wikis\n* create pages\n* edit pages\n* push to wikis\n* clone wikis\n\n#### webhook\n\nGives access to webhooks. This scope is required for any webhook-related operation.\n\nThis scope gives read access to existing webhook subscriptions on all\nresources the authorization mechanism can access, without needing further scopes. \nFor example:\n\n- A client can list all existing webhook subscriptions on a repository. The [`repository`](#repository) scope is not required.\n- Existing webhook subscriptions for the issue tracker on a repo can be retrieved without the [`issue`](#issue) scope. All that is required is the `webhook` scope.\n\nTo create webhooks, the client will need read access to the resource. Such as: for [`issue:created`](#issue-created), the client will need to\nhave both the `webhook` and the [`issue`](#issue) scope.\n\n* list webhook subscriptions on any accessible repository, user, team, or snippet\n* create/update/delete webhook subscriptions.\n\n#### snippet\n\nProvides read access to snippets.\nNo distinction is made between public and private snippets (public snippets are accessible without any form of authentication).\n\n* view any snippet\n* create snippet comments\n\n#### snippet:write\n\nProvides write access to snippets.\nNo distinction is made between public and private snippets (public snippets are accessible without any form of authentication).\nThis scope implicitly grants the [`snippet`](#snippet) scope which does not need to be requested separately.\n\n* create snippets\n* edit snippets\n* delete snippets\n\n#### email\n\nAbility to see the user's primary email address. This should make it easier to use Bitbucket Cloud as a login provider for apps or external applications.\n\n#### account\n\nAbility to see all the user's account information. Note that this doesn't include any ability to change any of the data.\n\n* see all email addresses\n* language\n* location\n* website\n* full name\n* SSH keys\n* user groups\n\n#### account:write\n\nAbility to change properties on the user's account.\n\n* delete the authorizing user's account\n* manage the user's groups\n* change a user's email addresses\n* change username, display name and avatar\n\n#### pipeline\n\nGives read-only access to pipelines, steps, deployment environments and variables.\n\n#### pipeline:write\n\nGives write access to pipelines. This scope allows a user to:\n* Stop pipelines\n* Rerun failed pipelines\n* Resume halted pipelines\n* Trigger manual pipelines.\n\nThis scope is not needed to trigger a build using a push. Performing a `git push` (or equivalent actions) will trigger the build. The token doing the push only needs the [`repository:write`](#repository-write) scope.\n\nThis doesn't give write access to create variables.\n\n#### pipeline:variable\n\nGives write access to create variables in pipelines at the various levels:\n* Workspace\n* Repository\n* Deployment\n\n#### runner\n\nGives read-only access to pipelines runners setup against a workspace or repository.\n\n#### runner:write\n\nGives write access to create/edit/disable/delete pipelines runners setup against a workspace or repository.\n\n### Basic auth\n\nBasic HTTP Authentication as per [RFC-2617](https://tools.ietf.org/html/rfc2617) (Digest not supported). Note that Basic Auth is available only with username and [app password](https://bitbucket.org/account/settings/app-passwords/) as credentials.\n\n### Repository Access Tokens\n\nRepository Access Tokens are passwords (or tokens) that provide access to\n_a single repository_. These tokens can authenticate with Bitbucket APIs for\nscripting, CI/CD tools, Bitbucket Cloud-connected apps, and Bitbucket Cloud\nintegrations. The level of access provided by the token is set when a repository\nadmin creates it, by setting permission scopes. Repository Access Tokens are\nlinked to their repository, not a user or a workspace, preventing them from\nbeing used to access any other repositories or workspaces.\n\nWhen using Bitbucket APIs with a Repository Access Token, the token will be\ntreated as the \"user\" in the Bitbucket UI and Bitbucket logs. This includes\nusing the Repository Access Token to leave a comment on a pull request, push a\ncommit, or merge a pull request. The Bitbucket UI and API responses will show\nthe Repository Access Token as a user. This user uses the Repository Access\nToken name and a custom icon to differentiate it from a regular user in the UI.\n\nFor details on creating, managing, and using Repository Access Tokens, visit\n[Repository Access Tokens](https://support.atlassian.com/bitbucket-cloud/docs/repository-access-tokens/).\n\n#### Considerations for using Repository Access Tokens\n\n* After creation, a Repository Access Token can't be viewed or modified. The\ntoken's name, created date, last accessed date, and scopes are visible on the\nRepository Access Token page.\n* Repository Access Tokens can only be granted a limited set of Bitbucket's \npermission scopes.\n* Provided you set the correct permission scopes, you can use a Repository Access\nToken to clone (`repository`) and push (`repository:write`) code to the\ntoken's corresponding repository.\n* You can't use a Repository Access Token to log into the Bitbucket website.\n* Repository Access Tokens don't require two-step verification.\n* You can set permission scopes (specific access rights) for each Repository\nAccess Token.\n* You can't use a Repository Access Token to manipulate or query repository\npermissions.\n* Repository Access Tokens will not be listed in any repository or workspace \npermission API response.\n* Repository Access Tokens are deactivated when a repository is transferred or deleted.\n* Any content created by the Repository Access Token will persist after the\nRepository Access Token has been revoked.\n\n#### Available permissions scopes\n\nThe available scopes for Repository Access Tokens are:\n\n- [repository](#repository)\n- [repository:write](#repository-write)\n- [repository:admin](#repository-admin)\n- [repository:delete](#repository-delete)\n- [pullrequest](#pullrequest)\n- [pullrequest:write](#pullrequest-write)\n- [webhook](#webhook)\n- [pipeline](#pipeline)\n- [pipeline:write](#pipeline-write)\n- [pipeline:variable](#pipeline-variable)\n- [runner](#runner)\n- [runner:write](#runner-write)\n\nThere are some APIs which are inaccessible for Repository Access Tokens, these are: \n\n* [Add a repository deploy key](/cloud/bitbucket/rest/api-group-deployments/#api-repositories-workspace-repo-slug-deploy-keys-post)\n* [Update a repository deploy key](/cloud/bitbucket/rest/api-group-deployments/#api-repositories-workspace-repo-slug-deploy-keys-key-id-put)\n* [Delete a repository deploy key](/cloud/bitbucket/rest/api-group-deployments/#api-repositories-workspace-repo-slug-deploy-keys-key-id-delete)\n\n### App passwords\n\nApp passwords allow users to make API calls to their Bitbucket account through apps such as Sourcetree.\n\nSome important points about app passwords:\n\n* You cannot view an app password or adjust permissions after you create the app password. Because app passwords are encrypted on our database and cannot be viewed by anyone. They are essentially designed to be disposable. If you need to change the scopes or lost the password just create a new one.\n* You cannot use them to log into your Bitbucket account.\n* You cannot use app passwords to manage team actions.\n\n App passwords are tied to an individual account's credentials and should not be shared. If you're sharing your app password you're essentially giving direct, authenticated, access to everything that password has been scoped to do with the Bitbucket API's.\n\n* You can use them for API call authentication, even if you don't have two-step verification enabled.\n* You can set permission scopes (specific access rights) for each app password.\n\n#### Create an app password\n\nTo create an app password:\n\n1. Select **Avatar > Bitbucket settings**.\n2. [Click **App passwords** in the Access management section.](https://bitbucket.org/account/settings/app-passwords/)\n3. Click **Create app password**.\n4. Give the app password a name related to the application that will use the password.\n5. Select the specific access and permissions you want this application password to have.\n6. Copy the generated password and either record or paste it into the application you want to give access. The password is only displayed this one time.\n\nThat's all there is to creating an app password. See your applications documentation for how to apply the app password for a specific application." }, { "anchor": "filtering", @@ -19723,7 +19809,7 @@ "state": { "type": "string", "description": "Provides some indication of the status of this commit", - "enum": ["FAILED", "SUCCESSFUL", "INPROGRESS", "STOPPED"] + "enum": ["INPROGRESS", "STOPPED", "FAILED", "SUCCESSFUL"] }, "updated_on": { "type": "string", @@ -20589,29 +20675,29 @@ "type": "string", "description": "The event identifier.", "enum": [ - "pullrequest:updated", - "issue:comment_created", - "issue:updated", - "repo:fork", - "pullrequest:changes_request_created", - "pullrequest:comment_created", - "repo:created", - "repo:updated", - "pullrequest:comment_deleted", - "pullrequest:rejected", "issue:created", - "repo:imported", - "pullrequest:unapproved", - "pullrequest:approved", - "pullrequest:comment_updated", - "project:updated", - "pullrequest:created", - "pullrequest:fulfilled", - "repo:commit_status_created", - "pullrequest:changes_request_removed", - "repo:push", - "repo:commit_comment_created", "repo:commit_status_updated", + "pullrequest:comment_created", + "pullrequest:created", + "pullrequest:unapproved", + "repo:commit_status_created", + "repo:created", + "pullrequest:changes_request_created", + "pullrequest:comment_updated", + "pullrequest:changes_request_removed", + "pullrequest:comment_deleted", + "repo:imported", + "repo:commit_comment_created", + "project:updated", + "repo:updated", + "issue:updated", + "pullrequest:rejected", + "pullrequest:fulfilled", + "issue:comment_created", + "repo:fork", + "pullrequest:updated", + "repo:push", + "pullrequest:approved", "repo:deleted", "repo:transfer" ] @@ -25356,29 +25442,29 @@ "items": { "type": "string", "enum": [ - "pullrequest:updated", - "issue:comment_created", - "issue:updated", - "repo:fork", - "pullrequest:changes_request_created", - "pullrequest:comment_created", - "repo:created", - "repo:updated", - "pullrequest:comment_deleted", - "pullrequest:rejected", "issue:created", - "repo:imported", - "pullrequest:unapproved", - "pullrequest:approved", - "pullrequest:comment_updated", - "project:updated", - "pullrequest:created", - "pullrequest:fulfilled", - "repo:commit_status_created", - "pullrequest:changes_request_removed", - "repo:push", - "repo:commit_comment_created", "repo:commit_status_updated", + "pullrequest:comment_created", + "pullrequest:created", + "pullrequest:unapproved", + "repo:commit_status_created", + "repo:created", + "pullrequest:changes_request_created", + "pullrequest:comment_updated", + "pullrequest:changes_request_removed", + "pullrequest:comment_deleted", + "repo:imported", + "repo:commit_comment_created", + "project:updated", + "repo:updated", + "issue:updated", + "pullrequest:rejected", + "pullrequest:fulfilled", + "issue:comment_created", + "repo:fork", + "pullrequest:updated", + "repo:push", + "pullrequest:approved", "repo:deleted", "repo:transfer" ] From 6f8716bd8ce11b929aa29f7ba0717630d7438ef0 Mon Sep 17 00:00:00 2001 From: Abhinav Rastogi Date: Fri, 9 Dec 2022 17:15:50 +0530 Subject: [PATCH 123/437] Adding option to pass additional headers in ProxiedSignInPage Signed-off-by: Abhinav Rastogi --- .../src/layout/ProxiedSignInPage/ProxiedSignInIdentity.ts | 4 +++- .../src/layout/ProxiedSignInPage/ProxiedSignInPage.tsx | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/core-components/src/layout/ProxiedSignInPage/ProxiedSignInIdentity.ts b/packages/core-components/src/layout/ProxiedSignInPage/ProxiedSignInIdentity.ts index 7633fc43c3..c8b4d66837 100644 --- a/packages/core-components/src/layout/ProxiedSignInPage/ProxiedSignInIdentity.ts +++ b/packages/core-components/src/layout/ProxiedSignInPage/ProxiedSignInIdentity.ts @@ -51,6 +51,7 @@ export function tokenToExpiry(jwtToken: string | undefined): Date { type ProxiedSignInIdentityOptions = { provider: string; discoveryApi: typeof discoveryApiRef.T; + getHeaders?: () => Promise; }; type State = @@ -192,6 +193,7 @@ export class ProxiedSignInIdentity implements IdentityApi { async fetchSession(): Promise { const baseUrl = await this.options.discoveryApi.getBaseUrl('auth'); + const headers = await this.options.getHeaders?.(); // Note that we do not use the fetchApi here, since this all happens before // sign-in completes so there can be no automatic token injection and @@ -200,7 +202,7 @@ export class ProxiedSignInIdentity implements IdentityApi { `${baseUrl}/${this.options.provider}/refresh`, { signal: this.abortController.signal, - headers: { 'x-requested-with': 'XMLHttpRequest' }, + headers: { ...headers, 'x-requested-with': 'XMLHttpRequest' }, credentials: 'include', }, ); diff --git a/packages/core-components/src/layout/ProxiedSignInPage/ProxiedSignInPage.tsx b/packages/core-components/src/layout/ProxiedSignInPage/ProxiedSignInPage.tsx index 965c603004..78ca5a8e84 100644 --- a/packages/core-components/src/layout/ProxiedSignInPage/ProxiedSignInPage.tsx +++ b/packages/core-components/src/layout/ProxiedSignInPage/ProxiedSignInPage.tsx @@ -36,6 +36,12 @@ export type ProxiedSignInPageProps = SignInPageProps & { * a properly configured auth provider ID in the auth backend. */ provider: string; + + /** + * An optional function which returns a promise resolving with any headers + * that need to be added to the call made to /refresh endpoint. + */ + getHeaders?: () => Promise; }; /** @@ -60,6 +66,7 @@ export const ProxiedSignInPage = (props: ProxiedSignInPageProps) => { const identity = new ProxiedSignInIdentity({ provider: props.provider, discoveryApi, + getHeaders: props.getHeaders, }); await identity.start(); From a5a2d12298b169768c5702ca20e1e9feeefbe986 Mon Sep 17 00:00:00 2001 From: Abhinav Rastogi Date: Fri, 9 Dec 2022 17:18:47 +0530 Subject: [PATCH 124/437] Added changeset Signed-off-by: Abhinav Rastogi --- .changeset/violet-dots-relate.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/violet-dots-relate.md diff --git a/.changeset/violet-dots-relate.md b/.changeset/violet-dots-relate.md new file mode 100644 index 0000000000..3b63129337 --- /dev/null +++ b/.changeset/violet-dots-relate.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-components': minor +--- + +Added option to pass additional headers in ProxiedSignInPage From 89035a0f58c50d4ae540fd7ced43fbfb383c1015 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 9 Dec 2022 14:18:25 +0100 Subject: [PATCH 125/437] catalog-backend: split up catalog database into 3 separate implementations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: Patrik Oldsberg --- plugins/catalog-backend/api-report.md | 2 +- .../database/DefaultCatalogDatabase.test.ts | 110 +++ .../src/database/DefaultCatalogDatabase.ts | 113 +++ .../DefaultProcessingDatabase.test.ts | 718 +----------------- .../src/database/DefaultProcessingDatabase.ts | 386 +--------- .../database/DefaultProviderDatabase.test.ts | 715 +++++++++++++++++ .../src/database/DefaultProviderDatabase.ts | 514 +++++++++++++ plugins/catalog-backend/src/database/types.ts | 46 +- .../catalog-backend/src/integration.test.ts | 14 +- .../src/processing/connectEntityProviders.ts | 12 +- .../catalog-backend/src/processing/types.ts | 10 +- .../service/AuthorizedEntitiesCatalog.test.ts | 2 + .../src/service/CatalogBuilder.ts | 14 +- .../src/service/DefaultRefreshService.test.ts | 39 +- .../src/service/DefaultRefreshService.ts | 6 +- 15 files changed, 1555 insertions(+), 1146 deletions(-) create mode 100644 plugins/catalog-backend/src/database/DefaultCatalogDatabase.test.ts create mode 100644 plugins/catalog-backend/src/database/DefaultCatalogDatabase.ts create mode 100644 plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts create mode 100644 plugins/catalog-backend/src/database/DefaultProviderDatabase.ts diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index a46abda5d3..3e6c5ef676 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -239,7 +239,7 @@ export type CatalogPermissionRule< // @alpha export const catalogPlugin: (options?: undefined) => BackendFeature; -// @public (undocumented) +// @public export interface CatalogProcessingEngine { // (undocumented) start(): Promise; diff --git a/plugins/catalog-backend/src/database/DefaultCatalogDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultCatalogDatabase.test.ts new file mode 100644 index 0000000000..b873f6a985 --- /dev/null +++ b/plugins/catalog-backend/src/database/DefaultCatalogDatabase.test.ts @@ -0,0 +1,110 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; +import { Logger } from 'winston'; +import { DefaultCatalogDatabase } from './DefaultCatalogDatabase'; +import { applyDatabaseMigrations } from './migrations'; +import { DbRefreshStateReferencesRow, DbRefreshStateRow } from './tables'; + +describe('DefaultCatalogDatabase', () => { + const defaultLogger = getVoidLogger(); + const databases = TestDatabases.create({ + ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + }); + + async function createDatabase( + databaseId: TestDatabaseId, + logger: Logger = defaultLogger, + ) { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + return { + knex, + db: new DefaultCatalogDatabase({ + database: knex, + logger, + }), + }; + } + + describe('listAncestors', () => { + let nextId = 1; + function makeEntity(ref: string) { + return { + entity_id: String(nextId++), + entity_ref: ref, + unprocessed_entity: JSON.stringify({ + kind: 'Location', + apiVersion: '1.0.0', + metadata: { + name: 'xyz', + }, + }), + errors: '[]', + next_update_at: '2019-01-01 23:00:00', + last_discovery_at: '2021-04-01 13:37:00', + }; + } + + it.each(databases.eachSupportedId())( + 'should return ancestors, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + + await knex('refresh_state').insert( + makeEntity('location:default/root-1'), + ); + await knex('refresh_state').insert( + makeEntity('location:default/root-2'), + ); + await knex('refresh_state').insert( + makeEntity('component:default/foobar'), + ); + + await knex( + 'refresh_state_references', + ).insert({ + source_key: 'source', + target_entity_ref: 'location:default/root-2', + }); + await knex( + 'refresh_state_references', + ).insert({ + source_entity_ref: 'location:default/root-2', + target_entity_ref: 'location:default/root-1', + }); + await knex( + 'refresh_state_references', + ).insert({ + source_entity_ref: 'location:default/root-1', + target_entity_ref: 'component:default/foobar', + }); + + const result = await db.transaction(tx => + db.listAncestors(tx, { + entityRef: 'component:default/foobar', + }), + ); + expect(result.entityRefs).toEqual([ + 'location:default/root-1', + 'location:default/root-2', + ]); + }, + ); + }); +}); diff --git a/plugins/catalog-backend/src/database/DefaultCatalogDatabase.ts b/plugins/catalog-backend/src/database/DefaultCatalogDatabase.ts new file mode 100644 index 0000000000..98b100c91e --- /dev/null +++ b/plugins/catalog-backend/src/database/DefaultCatalogDatabase.ts @@ -0,0 +1,113 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NotFoundError } from '@backstage/errors'; +import { Knex } from 'knex'; +import type { Logger } from 'winston'; +import { + CatalogDatabase, + ListAncestorsOptions, + ListAncestorsResult, + RefreshOptions, +} from './types'; +import { DbRefreshStateReferencesRow, DbRefreshStateRow } from './tables'; +import { rethrowError } from './conversion'; +import { Transaction } from './types'; + +const MAX_ANCESTOR_DEPTH = 32; + +export class DefaultCatalogDatabase implements CatalogDatabase { + constructor( + private readonly options: { + database: Knex; + logger: Logger; + }, + ) {} + + async transaction(fn: (tx: Transaction) => Promise): Promise { + try { + let result: T | undefined = undefined; + + await this.options.database.transaction( + async tx => { + // We can't return here, as knex swallows the return type in case the transaction is rolled back: + // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136 + result = await fn(tx); + }, + { + // If we explicitly trigger a rollback, don't fail. + doNotRejectOnRollback: true, + }, + ); + + return result!; + } catch (e) { + this.options.logger.debug(`Error during transaction, ${e}`); + throw rethrowError(e); + } + } + + async listAncestors( + txOpaque: Transaction, + options: ListAncestorsOptions, + ): Promise { + const tx = txOpaque as Knex.Transaction; + const { entityRef } = options; + const entityRefs = new Array(); + + let currentRef = entityRef.toLocaleLowerCase('en-US'); + for (let depth = 1; depth <= MAX_ANCESTOR_DEPTH; depth += 1) { + const rows = await tx( + 'refresh_state_references', + ) + .where({ target_entity_ref: currentRef }) + .select(); + + if (rows.length === 0) { + if (depth === 1) { + throw new NotFoundError(`Entity ${currentRef} not found`); + } + throw new NotFoundError( + `Entity ${entityRef} has a broken parent reference chain at ${currentRef}`, + ); + } + + const parentRef = rows.find(r => r.source_entity_ref)?.source_entity_ref; + if (!parentRef) { + // We've reached the top of the tree which is the entityProvider. + // In this case we refresh the entity itself. + return { entityRefs }; + } + entityRefs.push(parentRef); + currentRef = parentRef; + } + throw new Error( + `Unable receive ancestors for ${entityRef}, reached maximum depth of ${MAX_ANCESTOR_DEPTH}`, + ); + } + + async refresh(txOpaque: Transaction, options: RefreshOptions): Promise { + const tx = txOpaque as Knex.Transaction; + const { entityRef } = options; + + const updateResult = await tx('refresh_state') + .where({ entity_ref: entityRef.toLocaleLowerCase('en-US') }) + .update({ next_update_at: tx.fn.now() }); + if (updateResult === 0) { + throw new NotFoundError(`Failed to schedule ${entityRef} for refresh`); + } + } +} diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 1056557706..4fbb38b4fc 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -33,7 +33,7 @@ import { createRandomProcessingInterval } from '../processing/refresh'; import { timestampToDateTime } from './conversion'; import { generateStableHash } from './util'; -describe('Default Processing Database', () => { +describe('DefaultProcessingDatabase', () => { const defaultLogger = getVoidLogger(); const databases = TestDatabases.create({ ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], @@ -582,664 +582,6 @@ describe('Default Processing Database', () => { ); }); - describe('replaceUnprocessedEntities', () => { - const createLocations = async (db: Knex, entityRefs: string[]) => { - for (const ref of entityRefs) { - await insertRefreshStateRow(db, { - entity_id: uuid.v4(), - entity_ref: ref, - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: '2021-04-01 13:37:00', - last_discovery_at: '2021-04-01 13:37:00', - }); - } - }; - - it.each(databases.eachSupportedId())( - 'replaces all existing state correctly for simple dependency chains, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); - /* - config -> location:default/root -> location:default/root-1 -> location:default/root-2 - database -> location:default/second -> location:default/root-2 - */ - await createLocations(knex, [ - 'location:default/root', - 'location:default/root-1', - 'location:default/root-2', - 'location:default/second', - ]); - - await insertRefRow(knex, { - source_key: 'config', - target_entity_ref: 'location:default/root', - }); - - await insertRefRow(knex, { - source_key: 'database', - target_entity_ref: 'location:default/second', - }); - - await insertRefRow(knex, { - source_entity_ref: 'location:default/root', - target_entity_ref: 'location:default/root-1', - }); - - await insertRefRow(knex, { - source_entity_ref: 'location:default/root-1', - target_entity_ref: 'location:default/root-2', - }); - - await insertRefRow(knex, { - source_entity_ref: 'location:default/second', - target_entity_ref: 'location:default/root-2', - }); - - await db.transaction(tx => - db.replaceUnprocessedEntities(tx, { - type: 'full', - sourceKey: 'config', - items: [ - { - entity: { - apiVersion: '1.0.0', - metadata: { - name: 'new-root', - }, - kind: 'Location', - } as Entity, - locationKey: 'file:///tmp/foobar', - }, - ], - }), - ); - - const currentRefreshState = await knex( - 'refresh_state', - ).select(); - - const currentRefRowState = await knex( - 'refresh_state_references', - ).select(); - - for (const ref of [ - 'location:default/root', - 'location:default/root-1', - ]) { - expect( - currentRefreshState.some(t => t.entity_ref === ref), - ).toBeFalsy(); - } - - expect( - currentRefreshState.some( - t => t.entity_ref === 'location:default/new-root', - ), - ).toBeTruthy(); - - expect( - currentRefRowState.some( - t => - t.source_entity_ref === 'location:default/root' && - t.target_entity_ref === 'location:default/root-1', - ), - ).toBeFalsy(); - - expect( - currentRefRowState.some( - t => - t.source_entity_ref === 'location:default/root-1' && - t.target_entity_ref === 'location:default/root-2', - ), - ).toBeFalsy(); - - expect( - currentRefRowState.some( - t => - t.target_entity_ref === 'location:default/root-1' && - t.source_key === 'config', - ), - ).toBeFalsy(); - - expect( - currentRefRowState.some( - t => - t.target_entity_ref === 'location:default/new-root' && - t.source_key === 'config', - ), - ).toBeTruthy(); - }, - 60_000, - ); - - it.each(databases.eachSupportedId())( - 'should work for more complex chains, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); - /* - config -> location:default/root -> location:default/root-1 -> location:default/root-2 - config -> location:default/root -> location:default/root-1a -> location:default/root-2 - */ - await createLocations(knex, [ - 'location:default/root', - 'location:default/root-1', - 'location:default/root-2', - 'location:default/root-1a', - ]); - - await insertRefRow(knex, { - source_key: 'config', - target_entity_ref: 'location:default/root', - }); - - await insertRefRow(knex, { - source_entity_ref: 'location:default/root', - target_entity_ref: 'location:default/root-1', - }); - - await insertRefRow(knex, { - source_entity_ref: 'location:default/root', - target_entity_ref: 'location:default/root-1a', - }); - - await insertRefRow(knex, { - source_entity_ref: 'location:default/root-1', - target_entity_ref: 'location:default/root-2', - }); - - await insertRefRow(knex, { - source_entity_ref: 'location:default/root-1a', - target_entity_ref: 'location:default/root-2', - }); - - await db.transaction(async tx => { - await db.replaceUnprocessedEntities(tx, { - type: 'full', - sourceKey: 'config', - items: [ - { - entity: { - apiVersion: '1.0.0', - metadata: { - name: 'new-root', - }, - kind: 'Location', - } as Entity, - locationKey: 'file:/tmp/foobar', - }, - ], - }); - }); - - const currentRefreshState = await knex( - 'refresh_state', - ).select(); - - const currentRefRowState = await knex( - 'refresh_state_references', - ).select(); - - const deletedRefs = [ - 'location:default/root', - 'location:default/root-1', - 'location:default/root-1a', - 'location:default/root-2', - ]; - - for (const ref of deletedRefs) { - expect( - currentRefreshState.some(t => t.entity_ref === ref), - ).toBeFalsy(); - } - - expect( - currentRefreshState.some( - t => t.entity_ref === 'location:default/new-root', - ), - ).toBeTruthy(); - - expect( - currentRefRowState.some( - t => - t.source_key === 'config' && - t.target_entity_ref === 'location:default/new-root', - ), - ).toBeTruthy(); - - expect( - currentRefRowState.some( - t => - t.source_key === 'config' && - t.target_entity_ref === 'location:default/root', - ), - ).toBeFalsy(); - - expect( - currentRefRowState.some( - t => - t.source_entity_ref === 'location:default/root' && - t.target_entity_ref === 'location:default/root-1', - ), - ).toBeFalsy(); - - expect( - currentRefRowState.some( - t => - t.source_entity_ref === 'location:default/root' && - t.target_entity_ref === 'location:default/root-1a', - ), - ).toBeFalsy(); - - expect( - currentRefRowState.some( - t => - t.source_entity_ref === 'location:default/root-1' && - t.target_entity_ref === 'location:default/root-2', - ), - ).toBeFalsy(); - - expect( - currentRefRowState.some( - t => - t.source_entity_ref === 'location:default/root-1a' && - t.target_entity_ref === 'location:default/root-2', - ), - ).toBeFalsy(); - }, - 60_000, - ); - - it.each(databases.eachSupportedId())( - 'should add new locations using the delta options, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); - - // Existing state and references should stay - await createLocations(knex, ['location:default/existing']); - await insertRefRow(knex, { - source_key: 'lols', - target_entity_ref: 'location:default/existing', - }); - - await db.transaction(async tx => { - await db.replaceUnprocessedEntities(tx, { - type: 'delta', - sourceKey: 'lols', - removed: [], - added: [ - { - entity: { - apiVersion: '1.0.0', - metadata: { - name: 'new-root', - }, - kind: 'Location', - } as Entity, - locationKey: 'file:///tmp/foobar', - }, - ], - }); - }); - - const currentRefreshState = await knex( - 'refresh_state', - ).select(); - - const currentRefRowState = await knex( - 'refresh_state_references', - ).select(); - - expect( - currentRefreshState.some( - t => t.entity_ref === 'location:default/new-root', - ), - ).toBeTruthy(); - - expect( - currentRefRowState.some( - t => - t.source_key === 'lols' && - t.target_entity_ref === 'location:default/new-root', - ), - ).toBeTruthy(); - - expect( - currentRefreshState.some( - t => t.entity_ref === 'location:default/existing', - ), - ).toBeTruthy(); - - expect( - currentRefRowState.some( - t => - t.source_key === 'lols' && - t.target_entity_ref === 'location:default/existing', - ), - ).toBeTruthy(); - }, - 60_000, - ); - - it.each(databases.eachSupportedId())( - 'should not remove locations that are referenced elsewhere, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); - /* - config-1 -> location:default/root - config-2 -> location:default/root - */ - await createLocations(knex, ['location:default/root']); - - await insertRefRow(knex, { - source_key: 'config-1', - target_entity_ref: 'location:default/root', - }); - await insertRefRow(knex, { - source_key: 'config-2', - target_entity_ref: 'location:default/root', - }); - - await db.transaction(async tx => { - await db.replaceUnprocessedEntities(tx, { - type: 'full', - sourceKey: 'config-1', - items: [], - }); - }); - - const currentRefreshState = await knex( - 'refresh_state', - ).select(); - - const currentRefRowState = await knex( - 'refresh_state_references', - ).select(); - - expect(currentRefRowState).toEqual([ - expect.objectContaining({ - source_key: 'config-2', - target_entity_ref: 'location:default/root', - }), - ]); - - expect(currentRefreshState).toEqual([ - expect.objectContaining({ - entity_ref: 'location:default/root', - }), - ]); - }, - 60_000, - ); - - it.each(databases.eachSupportedId())( - 'should remove old locations using the delta options, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); - await createLocations(knex, ['location:default/new-root']); - - await insertRefRow(knex, { - source_key: 'lols', - target_entity_ref: 'location:default/new-root', - }); - - await db.transaction(async tx => { - await db.replaceUnprocessedEntities(tx, { - type: 'delta', - sourceKey: 'lols', - added: [], - removed: [ - { - entityRef: 'location:default/new-root', - locationKey: 'file:/tmp/foobar', - }, - ], - }); - }); - - const currentRefreshState = await knex( - 'refresh_state', - ).select(); - - const currentRefRowState = await knex( - 'refresh_state_references', - ).select(); - - expect( - currentRefreshState.some( - t => t.entity_ref === 'location:default/new-root', - ), - ).toBeFalsy(); - - expect( - currentRefRowState.some( - t => - t.source_key === 'lols' && - t.target_entity_ref === 'location:default/new-root', - ), - ).toBeFalsy(); - }, - 60_000, - ); - - it.each(databases.eachSupportedId())( - 'should update the location key during full replace, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); - await createLocations(knex, ['location:default/removed']); - await insertRefreshStateRow(knex, { - entity_id: uuid.v4(), - entity_ref: 'location:default/replaced', - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: '2021-04-01 13:37:00', - last_discovery_at: '2021-04-01 13:37:00', - location_key: 'file:///tmp/old', - }); - - await insertRefRow(knex, { - source_key: 'lols', - target_entity_ref: 'location:default/removed', - }); - await insertRefRow(knex, { - source_key: 'lols', - target_entity_ref: 'location:default/replaced', - }); - - await db.transaction(async tx => { - await db.replaceUnprocessedEntities(tx, { - type: 'full', - sourceKey: 'lols', - items: [ - { - entity: { - apiVersion: '1.0.0', - metadata: { - name: 'replaced', - }, - kind: 'Location', - } as Entity, - locationKey: 'file:///tmp/foobar', - }, - ], - }); - }); - - const currentRefreshState = await knex( - 'refresh_state', - ).select(); - expect(currentRefreshState).toEqual([ - expect.objectContaining({ - entity_ref: 'location:default/replaced', - location_key: 'file:///tmp/foobar', - }), - ]); - - const currentRefRowState = await knex( - 'refresh_state_references', - ).select(); - expect(currentRefRowState).toEqual([ - expect.objectContaining({ - source_key: 'lols', - target_entity_ref: 'location:default/replaced', - }), - ]); - }, - 60_000, - ); - - it.each(databases.eachSupportedId())( - 'should support replacing modified entities during a full update, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); - - await db.transaction(async tx => { - await db.replaceUnprocessedEntities(tx, { - type: 'full', - sourceKey: 'lols', - items: [ - { - entity: { - apiVersion: '1', - kind: 'Component', - metadata: { name: 'a' }, - spec: { marker: 'WILL_CHANGE' }, - } as Entity, - locationKey: 'file:///tmp/a', - }, - { - entity: { - apiVersion: '1', - kind: 'Component', - metadata: { name: 'b' }, - spec: { marker: 'NEVER_CHANGES' }, - } as Entity, - locationKey: 'file:///tmp/b', - }, - ], - }); - }); - - let state = await knex('refresh_state').select(); - expect(state).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - entity_ref: 'component:default/a', - location_key: 'file:///tmp/a', - unprocessed_entity: expect.stringContaining('WILL_CHANGE'), - }), - expect.objectContaining({ - entity_ref: 'component:default/b', - location_key: 'file:///tmp/b', - unprocessed_entity: expect.stringContaining('NEVER_CHANGES'), - }), - ]), - ); - - await db.transaction(async tx => { - await db.replaceUnprocessedEntities(tx, { - type: 'full', - sourceKey: 'lols', - items: [ - { - entity: { - apiVersion: '1', - kind: 'Component', - metadata: { name: 'a' }, - spec: { marker: 'HAS_CHANGED' }, - } as Entity, - locationKey: 'file:///tmp/a', - }, - { - entity: { - apiVersion: '1', - kind: 'Component', - metadata: { name: 'b' }, - spec: { marker: 'NEVER_CHANGES' }, - } as Entity, - locationKey: 'file:///tmp/b', - }, - ], - }); - }); - - state = await knex('refresh_state').select(); - expect(state).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - entity_ref: 'component:default/a', - location_key: 'file:///tmp/a', - unprocessed_entity: expect.stringContaining('HAS_CHANGED'), - }), - expect.objectContaining({ - entity_ref: 'component:default/b', - location_key: 'file:///tmp/b', - unprocessed_entity: expect.stringContaining('NEVER_CHANGES'), - }), - ]), - ); - }, - 60_000, - ); - - it.each(databases.eachSupportedId())( - 'should successfully fall back from batch to individual mode on conflicts, %p', - async databaseId => { - const fakeLogger = { - debug: jest.fn(), - }; - const { knex, db } = await createDatabase( - databaseId, - fakeLogger as any, - ); - - await createLocations(knex, ['component:default/a']); - - await insertRefRow(knex, { - source_key: undefined, - target_entity_ref: 'component:default/a', - }); - - await db.transaction(async tx => { - await db.replaceUnprocessedEntities(tx, { - type: 'full', - sourceKey: 'lols', - items: [ - { - entity: { - apiVersion: '1', - kind: 'Component', - metadata: { name: 'a' }, - spec: { marker: 'WILL_CHANGE' }, - } as Entity, - locationKey: 'file:///tmp/a', - }, - ], - }); - }); - expect(fakeLogger.debug).toHaveBeenCalledWith( - expect.stringMatching( - /Fast insert path failed, falling back to slow path/, - ), - ); - - const state = await knex('refresh_state').select(); - expect(state).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - entity_ref: 'component:default/a', - location_key: 'file:///tmp/a', - unprocessed_entity: expect.stringContaining('WILL_CHANGE'), - }), - ]), - ); - }, - 60_000, - ); - }); - describe('getProcessableEntities', () => { it.each(databases.eachSupportedId())( 'should return entities to process, %p', @@ -1329,64 +671,6 @@ describe('Default Processing Database', () => { ); }); - describe('listAncestors', () => { - let nextId = 1; - function makeEntity(ref: string) { - return { - entity_id: String(nextId++), - entity_ref: ref, - unprocessed_entity: JSON.stringify({ - kind: 'Location', - apiVersion: '1.0.0', - metadata: { - name: 'xyz', - }, - }), - errors: '[]', - next_update_at: '2019-01-01 23:00:00', - last_discovery_at: '2021-04-01 13:37:00', - }; - } - - it.each(databases.eachSupportedId())( - 'should return ancestors, %p', - async databaseId => { - const { knex, db } = await createDatabase(databaseId); - - await knex('refresh_state').insert( - makeEntity('location:default/root-1'), - ); - await knex('refresh_state').insert( - makeEntity('location:default/root-2'), - ); - await knex('refresh_state').insert( - makeEntity('component:default/foobar'), - ); - - await insertRefRow(knex, { - source_key: 'source', - target_entity_ref: 'location:default/root-2', - }); - await insertRefRow(knex, { - source_entity_ref: 'location:default/root-2', - target_entity_ref: 'location:default/root-1', - }); - await insertRefRow(knex, { - source_entity_ref: 'location:default/root-1', - target_entity_ref: 'component:default/foobar', - }); - - const result = await db.transaction(async tx => - db.listAncestors(tx, { entityRef: 'component:default/foobar' }), - ); - expect(result.entityRefs).toEqual([ - 'location:default/root-1', - 'location:default/root-2', - ]); - }, - ); - }); - describe('listParents', () => { let nextId = 1; function makeEntity(ref: string) { diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 97259bce29..c32f7d6f3d 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -15,7 +15,7 @@ */ import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; -import { ConflictError, NotFoundError } from '@backstage/errors'; +import { ConflictError } from '@backstage/errors'; import { Knex } from 'knex'; import lodash from 'lodash'; import { v4 as uuid } from 'uuid'; @@ -25,15 +25,10 @@ import { GetProcessableEntitiesResult, ProcessingDatabase, RefreshStateItem, - RefreshOptions, - ReplaceUnprocessedEntitiesOptions, UpdateProcessedEntityOptions, - ListAncestorsOptions, - ListAncestorsResult, UpdateEntityCacheOptions, ListParentsOptions, ListParentsResult, - RefreshByKeyOptions, } from './types'; import { ProcessingIntervalFunction } from '../processing/refresh'; import { rethrowError, timestampToDateTime } from './conversion'; @@ -54,7 +49,6 @@ import { DeferredEntity } from '@backstage/plugin-catalog-node'; // errors in the underlying engine due to exceeding query limits, but large // enough to get the speed benefits. const BATCH_SIZE = 50; -const MAX_ANCESTOR_DEPTH = 32; export class DefaultProcessingDatabase implements ProcessingDatabase { constructor( @@ -193,236 +187,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { .where('entity_id', id); } - async replaceUnprocessedEntities( - txOpaque: Transaction, - options: ReplaceUnprocessedEntitiesOptions, - ): Promise { - const tx = txOpaque as Knex.Transaction; - - const { toAdd, toUpsert, toRemove } = await this.createDelta(tx, options); - - if (toRemove.length) { - let removedCount = 0; - const rootId = () => { - if (tx.client.config.client.includes('mysql')) { - return tx.raw('CAST(NULL as UNSIGNED INT)', []); - } - - return tx.raw('CAST(NULL as INT)', []); - }; - for (const refs of lodash.chunk(toRemove, 1000)) { - /* - WITH RECURSIVE - -- All the nodes that can be reached downwards from our root - descendants(root_id, entity_ref) AS ( - SELECT id, target_entity_ref - FROM refresh_state_references - WHERE source_key = "R1" AND target_entity_ref = "A" - UNION - SELECT descendants.root_id, target_entity_ref - FROM descendants - JOIN refresh_state_references ON source_entity_ref = descendants.entity_ref - ), - -- All the nodes that can be reached upwards from the descendants - ancestors(root_id, via_entity_ref, to_entity_ref) AS ( - SELECT CAST(NULL as INT), entity_ref, entity_ref - FROM descendants - UNION - SELECT - CASE WHEN source_key IS NOT NULL THEN id ELSE NULL END, - source_entity_ref, - ancestors.to_entity_ref - FROM ancestors - JOIN refresh_state_references ON target_entity_ref = ancestors.via_entity_ref - ) - -- Start out with all of the descendants - SELECT descendants.entity_ref - FROM descendants - -- Expand with all ancestors that point to those, but aren't the current root - LEFT OUTER JOIN ancestors - ON ancestors.to_entity_ref = descendants.entity_ref - AND ancestors.root_id IS NOT NULL - AND ancestors.root_id != descendants.root_id - -- Exclude all lines that had such a foreign ancestor - WHERE ancestors.root_id IS NULL; - */ - removedCount += await tx('refresh_state') - .whereIn('entity_ref', function orphanedEntityRefs(orphans) { - return ( - orphans - // All the nodes that can be reached downwards from our root - .withRecursive('descendants', function descendants(outer) { - return outer - .select({ root_id: 'id', entity_ref: 'target_entity_ref' }) - .from('refresh_state_references') - .where('source_key', options.sourceKey) - .whereIn('target_entity_ref', refs) - .union(function recursive(inner) { - return inner - .select({ - root_id: 'descendants.root_id', - entity_ref: - 'refresh_state_references.target_entity_ref', - }) - .from('descendants') - .join('refresh_state_references', { - 'descendants.entity_ref': - 'refresh_state_references.source_entity_ref', - }); - }); - }) - // All the nodes that can be reached upwards from the descendants - .withRecursive('ancestors', function ancestors(outer) { - return outer - .select({ - root_id: rootId(), - via_entity_ref: 'entity_ref', - to_entity_ref: 'entity_ref', - }) - .from('descendants') - .union(function recursive(inner) { - return inner - .select({ - root_id: tx.raw( - 'CASE WHEN source_key IS NOT NULL THEN id ELSE NULL END', - [], - ), - via_entity_ref: 'source_entity_ref', - to_entity_ref: 'ancestors.to_entity_ref', - }) - .from('ancestors') - .join('refresh_state_references', { - target_entity_ref: 'ancestors.via_entity_ref', - }); - }); - }) - // Start out with all of the descendants - .select('descendants.entity_ref') - .from('descendants') - // Expand with all ancestors that point to those, but aren't the current root - .leftOuterJoin('ancestors', function keepaliveRoots() { - this.on( - 'ancestors.to_entity_ref', - '=', - 'descendants.entity_ref', - ); - this.andOnNotNull('ancestors.root_id'); - this.andOn('ancestors.root_id', '!=', 'descendants.root_id'); - }) - .whereNull('ancestors.root_id') - ); - }) - .delete(); - - await tx('refresh_state_references') - .where('source_key', '=', options.sourceKey) - .whereIn('target_entity_ref', refs) - .delete(); - } - - this.options.logger.debug( - `removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`, - ); - } - - if (toAdd.length) { - // The reason for this chunking, rather than just massively batch - // inserting the entire payload, is that we fall back to the individual - // upsert mechanism below on conflicts. That path is massively slower than - // the fast batch path, so we don't want to end up accidentally having to - // for example item-by-item upsert tens of thousands of entities in a - // large initial delivery dump. The implication is that the size of these - // chunks needs to weigh the benefit of fast successful inserts, against - // the drawback of super slow but more rare fallbacks. There's quickly - // diminishing returns though with turning up this value way high. - for (const chunk of lodash.chunk(toAdd, 50)) { - try { - await tx.batchInsert( - 'refresh_state', - chunk.map(item => ({ - entity_id: uuid(), - entity_ref: stringifyEntityRef(item.deferred.entity), - unprocessed_entity: JSON.stringify(item.deferred.entity), - unprocessed_hash: item.hash, - errors: '', - location_key: item.deferred.locationKey, - next_update_at: tx.fn.now(), - last_discovery_at: tx.fn.now(), - })), - BATCH_SIZE, - ); - await tx.batchInsert( - 'refresh_state_references', - chunk.map(item => ({ - source_key: options.sourceKey, - target_entity_ref: stringifyEntityRef(item.deferred.entity), - })), - BATCH_SIZE, - ); - } catch (error) { - if (!isDatabaseConflictError(error)) { - throw error; - } else { - this.options.logger.debug( - `Fast insert path failed, falling back to slow path, ${error}`, - ); - toUpsert.push(...chunk); - } - } - } - } - - if (toUpsert.length) { - for (const { - deferred: { entity, locationKey }, - hash, - } of toUpsert) { - const entityRef = stringifyEntityRef(entity); - - try { - let ok = await this.updateUnprocessedEntity( - tx, - entity, - hash, - locationKey, - ); - if (!ok) { - ok = await this.insertUnprocessedEntity( - tx, - entity, - hash, - locationKey, - ); - } - - if (ok) { - await tx( - 'refresh_state_references', - ).insert({ - source_key: options.sourceKey, - target_entity_ref: entityRef, - }); - } else { - const conflictingKey = await this.checkLocationKeyConflict( - tx, - entityRef, - locationKey, - ); - if (conflictingKey) { - this.options.logger.warn( - `Source ${options.sourceKey} detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`, - ); - } - } - } catch (error) { - this.options.logger.error( - `Failed to add '${entityRef}' from source '${options.sourceKey}', ${error}`, - ); - } - } - } - } - async getProcessableEntities( txOpaque: Transaction, request: { processBatchSize: number }, @@ -487,45 +251,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { }; } - async listAncestors( - txOpaque: Transaction, - options: ListAncestorsOptions, - ): Promise { - const tx = txOpaque as Knex.Transaction; - const { entityRef } = options; - const entityRefs = new Array(); - - let currentRef = entityRef.toLocaleLowerCase('en-US'); - for (let depth = 1; depth <= MAX_ANCESTOR_DEPTH; depth += 1) { - const rows = await tx( - 'refresh_state_references', - ) - .where({ target_entity_ref: currentRef }) - .select(); - - if (rows.length === 0) { - if (depth === 1) { - throw new NotFoundError(`Entity ${currentRef} not found`); - } - throw new NotFoundError( - `Entity ${entityRef} has a broken parent reference chain at ${currentRef}`, - ); - } - - const parentRef = rows.find(r => r.source_entity_ref)?.source_entity_ref; - if (!parentRef) { - // We've reached the top of the tree which is the entityProvider. - // In this case we refresh the entity itself. - return { entityRefs }; - } - entityRefs.push(parentRef); - currentRef = parentRef; - } - throw new Error( - `Unable receive ancestors for ${entityRef}, reached maximum depth of ${MAX_ANCESTOR_DEPTH}`, - ); - } - async listParents( txOpaque: Transaction, options: ListParentsOptions, @@ -543,37 +268,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { return { entityRefs }; } - async refresh(txOpaque: Transaction, options: RefreshOptions): Promise { - const tx = txOpaque as Knex.Transaction; - const { entityRef } = options; - - const updateResult = await tx('refresh_state') - .where({ entity_ref: entityRef.toLocaleLowerCase('en-US') }) - .update({ next_update_at: tx.fn.now() }); - if (updateResult === 0) { - throw new NotFoundError(`Failed to schedule ${entityRef} for refresh`); - } - } - - async refreshByRefreshKeys( - txOpaque: Transaction, - options: RefreshByKeyOptions, - ) { - const tx = txOpaque as Knex.Transaction; - const { keys } = options; - - await tx('refresh_state') - .whereIn('entity_id', function selectEntityRefs(tx2) { - tx2 - .whereIn('key', keys) - .select({ - entity_id: 'refresh_keys.entity_id', - }) - .from('refresh_keys'); - }) - .update({ next_update_at: tx.fn.now() }); - } - async transaction(fn: (tx: Transaction) => Promise): Promise { try { let result: T | undefined = undefined; @@ -721,84 +415,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { ); } - private async createDelta( - tx: Knex.Transaction, - options: ReplaceUnprocessedEntitiesOptions, - ): Promise<{ - toAdd: { deferred: DeferredEntity; hash: string }[]; - toUpsert: { deferred: DeferredEntity; hash: string }[]; - toRemove: string[]; - }> { - if (options.type === 'delta') { - return { - toAdd: [], - toUpsert: options.added.map(e => ({ - deferred: e, - hash: generateStableHash(e.entity), - })), - toRemove: options.removed.map(e => e.entityRef), - }; - } - - // Grab all of the existing references from the same source, and their locationKeys as well - const oldRefs = await tx( - 'refresh_state_references', - ) - .leftJoin('refresh_state', { - target_entity_ref: 'entity_ref', - }) - .where({ source_key: options.sourceKey }) - .select({ - target_entity_ref: 'refresh_state_references.target_entity_ref', - location_key: 'refresh_state.location_key', - unprocessed_hash: 'refresh_state.unprocessed_hash', - }); - - const items = options.items.map(deferred => ({ - deferred, - ref: stringifyEntityRef(deferred.entity), - hash: generateStableHash(deferred.entity), - })); - - const oldRefsSet = new Map( - oldRefs.map(r => [ - r.target_entity_ref, - { - locationKey: r.location_key, - oldEntityHash: r.unprocessed_hash, - }, - ]), - ); - const newRefsSet = new Set(items.map(item => item.ref)); - - const toAdd = new Array<{ deferred: DeferredEntity; hash: string }>(); - const toUpsert = new Array<{ deferred: DeferredEntity; hash: string }>(); - const toRemove = oldRefs - .map(row => row.target_entity_ref) - .filter(ref => !newRefsSet.has(ref)); - - for (const item of items) { - const oldRef = oldRefsSet.get(item.ref); - const upsertItem = { deferred: item.deferred, hash: item.hash }; - if (!oldRef) { - // Add any entity that does not exist in the database - toAdd.push(upsertItem); - } else if ( - (oldRef?.locationKey ?? undefined) !== - (item.deferred.locationKey ?? undefined) - ) { - // Remove and then re-add any entity that exists, but with a different location key - toRemove.push(item.ref); - toAdd.push(upsertItem); - } else if (oldRef.oldEntityHash !== item.hash) { - // Entities with modifications should be pushed through too - toUpsert.push(upsertItem); - } - } - - return { toAdd, toUpsert, toRemove }; - } - /** * Add a set of deferred entities for processing. * The entities will be added at the front of the processing queue. diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts new file mode 100644 index 0000000000..43f2209b60 --- /dev/null +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts @@ -0,0 +1,715 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; +import { Entity } from '@backstage/catalog-model'; +import { Knex } from 'knex'; +import * as uuid from 'uuid'; +import { Logger } from 'winston'; +import { DefaultProviderDatabase } from './DefaultProviderDatabase'; +import { applyDatabaseMigrations } from './migrations'; +import { DbRefreshStateReferencesRow, DbRefreshStateRow } from './tables'; + +describe('DefaultProviderDatabase', () => { + const defaultLogger = getVoidLogger(); + const databases = TestDatabases.create({ + ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + }); + + async function createDatabase( + databaseId: TestDatabaseId, + logger: Logger = defaultLogger, + ) { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + return { + knex, + db: new DefaultProviderDatabase({ + database: knex, + logger, + }), + }; + } + + const insertRefRow = async (db: Knex, ref: DbRefreshStateReferencesRow) => { + return db('refresh_state_references').insert( + ref, + ); + }; + + const insertRefreshStateRow = async (db: Knex, ref: DbRefreshStateRow) => { + await db('refresh_state').insert(ref); + }; + + describe('replaceUnprocessedEntities', () => { + const createLocations = async (db: Knex, entityRefs: string[]) => { + for (const ref of entityRefs) { + await insertRefreshStateRow(db, { + entity_id: uuid.v4(), + entity_ref: ref, + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + } + }; + + it.each(databases.eachSupportedId())( + 'replaces all existing state correctly for simple dependency chains, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + /* + config -> location:default/root -> location:default/root-1 -> location:default/root-2 + database -> location:default/second -> location:default/root-2 + */ + await createLocations(knex, [ + 'location:default/root', + 'location:default/root-1', + 'location:default/root-2', + 'location:default/second', + ]); + + await insertRefRow(knex, { + source_key: 'config', + target_entity_ref: 'location:default/root', + }); + + await insertRefRow(knex, { + source_key: 'database', + target_entity_ref: 'location:default/second', + }); + + await insertRefRow(knex, { + source_entity_ref: 'location:default/root', + target_entity_ref: 'location:default/root-1', + }); + + await insertRefRow(knex, { + source_entity_ref: 'location:default/root-1', + target_entity_ref: 'location:default/root-2', + }); + + await insertRefRow(knex, { + source_entity_ref: 'location:default/second', + target_entity_ref: 'location:default/root-2', + }); + + await db.transaction(tx => + db.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'config', + items: [ + { + entity: { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + locationKey: 'file:///tmp/foobar', + }, + ], + }), + ); + + const currentRefreshState = await knex( + 'refresh_state', + ).select(); + + const currentRefRowState = await knex( + 'refresh_state_references', + ).select(); + + for (const ref of [ + 'location:default/root', + 'location:default/root-1', + ]) { + expect( + currentRefreshState.some(t => t.entity_ref === ref), + ).toBeFalsy(); + } + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root' && + t.target_entity_ref === 'location:default/root-1', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root-1' && + t.target_entity_ref === 'location:default/root-2', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.target_entity_ref === 'location:default/root-1' && + t.source_key === 'config', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.target_entity_ref === 'location:default/new-root' && + t.source_key === 'config', + ), + ).toBeTruthy(); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should work for more complex chains, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + /* + config -> location:default/root -> location:default/root-1 -> location:default/root-2 + config -> location:default/root -> location:default/root-1a -> location:default/root-2 + */ + await createLocations(knex, [ + 'location:default/root', + 'location:default/root-1', + 'location:default/root-2', + 'location:default/root-1a', + ]); + + await insertRefRow(knex, { + source_key: 'config', + target_entity_ref: 'location:default/root', + }); + + await insertRefRow(knex, { + source_entity_ref: 'location:default/root', + target_entity_ref: 'location:default/root-1', + }); + + await insertRefRow(knex, { + source_entity_ref: 'location:default/root', + target_entity_ref: 'location:default/root-1a', + }); + + await insertRefRow(knex, { + source_entity_ref: 'location:default/root-1', + target_entity_ref: 'location:default/root-2', + }); + + await insertRefRow(knex, { + source_entity_ref: 'location:default/root-1a', + target_entity_ref: 'location:default/root-2', + }); + + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'config', + items: [ + { + entity: { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + locationKey: 'file:/tmp/foobar', + }, + ], + }); + }); + + const currentRefreshState = await knex( + 'refresh_state', + ).select(); + + const currentRefRowState = await knex( + 'refresh_state_references', + ).select(); + + const deletedRefs = [ + 'location:default/root', + 'location:default/root-1', + 'location:default/root-1a', + 'location:default/root-2', + ]; + + for (const ref of deletedRefs) { + expect( + currentRefreshState.some(t => t.entity_ref === ref), + ).toBeFalsy(); + } + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'config' && + t.target_entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'config' && + t.target_entity_ref === 'location:default/root', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root' && + t.target_entity_ref === 'location:default/root-1', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root' && + t.target_entity_ref === 'location:default/root-1a', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root-1' && + t.target_entity_ref === 'location:default/root-2', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root-1a' && + t.target_entity_ref === 'location:default/root-2', + ), + ).toBeFalsy(); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should add new locations using the delta options, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + + // Existing state and references should stay + await createLocations(knex, ['location:default/existing']); + await insertRefRow(knex, { + source_key: 'lols', + target_entity_ref: 'location:default/existing', + }); + + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + type: 'delta', + sourceKey: 'lols', + removed: [], + added: [ + { + entity: { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + locationKey: 'file:///tmp/foobar', + }, + ], + }); + }); + + const currentRefreshState = await knex( + 'refresh_state', + ).select(); + + const currentRefRowState = await knex( + 'refresh_state_references', + ).select(); + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'lols' && + t.target_entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/existing', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'lols' && + t.target_entity_ref === 'location:default/existing', + ), + ).toBeTruthy(); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should not remove locations that are referenced elsewhere, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + /* + config-1 -> location:default/root + config-2 -> location:default/root + */ + await createLocations(knex, ['location:default/root']); + + await insertRefRow(knex, { + source_key: 'config-1', + target_entity_ref: 'location:default/root', + }); + await insertRefRow(knex, { + source_key: 'config-2', + target_entity_ref: 'location:default/root', + }); + + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'config-1', + items: [], + }); + }); + + const currentRefreshState = await knex( + 'refresh_state', + ).select(); + + const currentRefRowState = await knex( + 'refresh_state_references', + ).select(); + + expect(currentRefRowState).toEqual([ + expect.objectContaining({ + source_key: 'config-2', + target_entity_ref: 'location:default/root', + }), + ]); + + expect(currentRefreshState).toEqual([ + expect.objectContaining({ + entity_ref: 'location:default/root', + }), + ]); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should remove old locations using the delta options, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + await createLocations(knex, ['location:default/new-root']); + + await insertRefRow(knex, { + source_key: 'lols', + target_entity_ref: 'location:default/new-root', + }); + + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + type: 'delta', + sourceKey: 'lols', + added: [], + removed: [ + { + entityRef: 'location:default/new-root', + locationKey: 'file:/tmp/foobar', + }, + ], + }); + }); + + const currentRefreshState = await knex( + 'refresh_state', + ).select(); + + const currentRefRowState = await knex( + 'refresh_state_references', + ).select(); + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'lols' && + t.target_entity_ref === 'location:default/new-root', + ), + ).toBeFalsy(); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should update the location key during full replace, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + await createLocations(knex, ['location:default/removed']); + await insertRefreshStateRow(knex, { + entity_id: uuid.v4(), + entity_ref: 'location:default/replaced', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + location_key: 'file:///tmp/old', + }); + + await insertRefRow(knex, { + source_key: 'lols', + target_entity_ref: 'location:default/removed', + }); + await insertRefRow(knex, { + source_key: 'lols', + target_entity_ref: 'location:default/replaced', + }); + + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'lols', + items: [ + { + entity: { + apiVersion: '1.0.0', + metadata: { + name: 'replaced', + }, + kind: 'Location', + } as Entity, + locationKey: 'file:///tmp/foobar', + }, + ], + }); + }); + + const currentRefreshState = await knex( + 'refresh_state', + ).select(); + expect(currentRefreshState).toEqual([ + expect.objectContaining({ + entity_ref: 'location:default/replaced', + location_key: 'file:///tmp/foobar', + }), + ]); + + const currentRefRowState = await knex( + 'refresh_state_references', + ).select(); + expect(currentRefRowState).toEqual([ + expect.objectContaining({ + source_key: 'lols', + target_entity_ref: 'location:default/replaced', + }), + ]); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should support replacing modified entities during a full update, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'lols', + items: [ + { + entity: { + apiVersion: '1', + kind: 'Component', + metadata: { name: 'a' }, + spec: { marker: 'WILL_CHANGE' }, + } as Entity, + locationKey: 'file:///tmp/a', + }, + { + entity: { + apiVersion: '1', + kind: 'Component', + metadata: { name: 'b' }, + spec: { marker: 'NEVER_CHANGES' }, + } as Entity, + locationKey: 'file:///tmp/b', + }, + ], + }); + }); + + let state = await knex('refresh_state').select(); + expect(state).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + entity_ref: 'component:default/a', + location_key: 'file:///tmp/a', + unprocessed_entity: expect.stringContaining('WILL_CHANGE'), + }), + expect.objectContaining({ + entity_ref: 'component:default/b', + location_key: 'file:///tmp/b', + unprocessed_entity: expect.stringContaining('NEVER_CHANGES'), + }), + ]), + ); + + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'lols', + items: [ + { + entity: { + apiVersion: '1', + kind: 'Component', + metadata: { name: 'a' }, + spec: { marker: 'HAS_CHANGED' }, + } as Entity, + locationKey: 'file:///tmp/a', + }, + { + entity: { + apiVersion: '1', + kind: 'Component', + metadata: { name: 'b' }, + spec: { marker: 'NEVER_CHANGES' }, + } as Entity, + locationKey: 'file:///tmp/b', + }, + ], + }); + }); + + state = await knex('refresh_state').select(); + expect(state).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + entity_ref: 'component:default/a', + location_key: 'file:///tmp/a', + unprocessed_entity: expect.stringContaining('HAS_CHANGED'), + }), + expect.objectContaining({ + entity_ref: 'component:default/b', + location_key: 'file:///tmp/b', + unprocessed_entity: expect.stringContaining('NEVER_CHANGES'), + }), + ]), + ); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should successfully fall back from batch to individual mode on conflicts, %p', + async databaseId => { + const fakeLogger = { + debug: jest.fn(), + }; + const { knex, db } = await createDatabase( + databaseId, + fakeLogger as any, + ); + + await createLocations(knex, ['component:default/a']); + + await insertRefRow(knex, { + source_key: undefined, + target_entity_ref: 'component:default/a', + }); + + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'lols', + items: [ + { + entity: { + apiVersion: '1', + kind: 'Component', + metadata: { name: 'a' }, + spec: { marker: 'WILL_CHANGE' }, + } as Entity, + locationKey: 'file:///tmp/a', + }, + ], + }); + }); + expect(fakeLogger.debug).toHaveBeenCalledWith( + expect.stringMatching( + /Fast insert path failed, falling back to slow path/, + ), + ); + + const state = await knex('refresh_state').select(); + expect(state).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + entity_ref: 'component:default/a', + location_key: 'file:///tmp/a', + unprocessed_entity: expect.stringContaining('WILL_CHANGE'), + }), + ]), + ); + }, + 60_000, + ); + }); +}); diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts new file mode 100644 index 0000000000..05a1011c8d --- /dev/null +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts @@ -0,0 +1,514 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isDatabaseConflictError } from '@backstage/backend-common'; +import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; +import { DeferredEntity } from '@backstage/plugin-catalog-node'; +import { Knex } from 'knex'; +import lodash from 'lodash'; +import { v4 as uuid } from 'uuid'; +import type { Logger } from 'winston'; +import { rethrowError } from './conversion'; +import { DbRefreshStateReferencesRow, DbRefreshStateRow } from './tables'; +import { + ProviderDatabase, + RefreshByKeyOptions, + ReplaceUnprocessedEntitiesOptions, + Transaction, +} from './types'; +import { generateStableHash } from './util'; + +// The number of items that are sent per batch to the database layer, when +// doing .batchInsert calls to knex. This needs to be low enough to not cause +// errors in the underlying engine due to exceeding query limits, but large +// enough to get the speed benefits. +const BATCH_SIZE = 50; + +export class DefaultProviderDatabase implements ProviderDatabase { + constructor( + private readonly options: { + database: Knex; + logger: Logger; + }, + ) {} + + async transaction(fn: (tx: Transaction) => Promise): Promise { + try { + let result: T | undefined = undefined; + + await this.options.database.transaction( + async tx => { + // We can't return here, as knex swallows the return type in case the transaction is rolled back: + // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136 + result = await fn(tx); + }, + { + // If we explicitly trigger a rollback, don't fail. + doNotRejectOnRollback: true, + }, + ); + + return result!; + } catch (e) { + this.options.logger.debug(`Error during transaction, ${e}`); + throw rethrowError(e); + } + } + + async replaceUnprocessedEntities( + txOpaque: Transaction, + options: ReplaceUnprocessedEntitiesOptions, + ): Promise { + const tx = txOpaque as Knex.Transaction; + + const { toAdd, toUpsert, toRemove } = await this.createDelta(tx, options); + + if (toRemove.length) { + let removedCount = 0; + const rootId = () => { + if (tx.client.config.client.includes('mysql')) { + return tx.raw('CAST(NULL as UNSIGNED INT)', []); + } + + return tx.raw('CAST(NULL as INT)', []); + }; + for (const refs of lodash.chunk(toRemove, 1000)) { + /* + WITH RECURSIVE + -- All the nodes that can be reached downwards from our root + descendants(root_id, entity_ref) AS ( + SELECT id, target_entity_ref + FROM refresh_state_references + WHERE source_key = "R1" AND target_entity_ref = "A" + UNION + SELECT descendants.root_id, target_entity_ref + FROM descendants + JOIN refresh_state_references ON source_entity_ref = descendants.entity_ref + ), + -- All the nodes that can be reached upwards from the descendants + ancestors(root_id, via_entity_ref, to_entity_ref) AS ( + SELECT CAST(NULL as INT), entity_ref, entity_ref + FROM descendants + UNION + SELECT + CASE WHEN source_key IS NOT NULL THEN id ELSE NULL END, + source_entity_ref, + ancestors.to_entity_ref + FROM ancestors + JOIN refresh_state_references ON target_entity_ref = ancestors.via_entity_ref + ) + -- Start out with all of the descendants + SELECT descendants.entity_ref + FROM descendants + -- Expand with all ancestors that point to those, but aren't the current root + LEFT OUTER JOIN ancestors + ON ancestors.to_entity_ref = descendants.entity_ref + AND ancestors.root_id IS NOT NULL + AND ancestors.root_id != descendants.root_id + -- Exclude all lines that had such a foreign ancestor + WHERE ancestors.root_id IS NULL; + */ + removedCount += await tx('refresh_state') + .whereIn('entity_ref', function orphanedEntityRefs(orphans) { + return ( + orphans + // All the nodes that can be reached downwards from our root + .withRecursive('descendants', function descendants(outer) { + return outer + .select({ root_id: 'id', entity_ref: 'target_entity_ref' }) + .from('refresh_state_references') + .where('source_key', options.sourceKey) + .whereIn('target_entity_ref', refs) + .union(function recursive(inner) { + return inner + .select({ + root_id: 'descendants.root_id', + entity_ref: + 'refresh_state_references.target_entity_ref', + }) + .from('descendants') + .join('refresh_state_references', { + 'descendants.entity_ref': + 'refresh_state_references.source_entity_ref', + }); + }); + }) + // All the nodes that can be reached upwards from the descendants + .withRecursive('ancestors', function ancestors(outer) { + return outer + .select({ + root_id: rootId(), + via_entity_ref: 'entity_ref', + to_entity_ref: 'entity_ref', + }) + .from('descendants') + .union(function recursive(inner) { + return inner + .select({ + root_id: tx.raw( + 'CASE WHEN source_key IS NOT NULL THEN id ELSE NULL END', + [], + ), + via_entity_ref: 'source_entity_ref', + to_entity_ref: 'ancestors.to_entity_ref', + }) + .from('ancestors') + .join('refresh_state_references', { + target_entity_ref: 'ancestors.via_entity_ref', + }); + }); + }) + // Start out with all of the descendants + .select('descendants.entity_ref') + .from('descendants') + // Expand with all ancestors that point to those, but aren't the current root + .leftOuterJoin('ancestors', function keepaliveRoots() { + this.on( + 'ancestors.to_entity_ref', + '=', + 'descendants.entity_ref', + ); + this.andOnNotNull('ancestors.root_id'); + this.andOn('ancestors.root_id', '!=', 'descendants.root_id'); + }) + .whereNull('ancestors.root_id') + ); + }) + .delete(); + + await tx('refresh_state_references') + .where('source_key', '=', options.sourceKey) + .whereIn('target_entity_ref', refs) + .delete(); + } + + this.options.logger.debug( + `removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`, + ); + } + + if (toAdd.length) { + // The reason for this chunking, rather than just massively batch + // inserting the entire payload, is that we fall back to the individual + // upsert mechanism below on conflicts. That path is massively slower than + // the fast batch path, so we don't want to end up accidentally having to + // for example item-by-item upsert tens of thousands of entities in a + // large initial delivery dump. The implication is that the size of these + // chunks needs to weigh the benefit of fast successful inserts, against + // the drawback of super slow but more rare fallbacks. There's quickly + // diminishing returns though with turning up this value way high. + for (const chunk of lodash.chunk(toAdd, 50)) { + try { + await tx.batchInsert( + 'refresh_state', + chunk.map(item => ({ + entity_id: uuid(), + entity_ref: stringifyEntityRef(item.deferred.entity), + unprocessed_entity: JSON.stringify(item.deferred.entity), + unprocessed_hash: item.hash, + errors: '', + location_key: item.deferred.locationKey, + next_update_at: tx.fn.now(), + last_discovery_at: tx.fn.now(), + })), + BATCH_SIZE, + ); + await tx.batchInsert( + 'refresh_state_references', + chunk.map(item => ({ + source_key: options.sourceKey, + target_entity_ref: stringifyEntityRef(item.deferred.entity), + })), + BATCH_SIZE, + ); + } catch (error) { + if (!isDatabaseConflictError(error)) { + throw error; + } else { + this.options.logger.debug( + `Fast insert path failed, falling back to slow path, ${error}`, + ); + toUpsert.push(...chunk); + } + } + } + } + + if (toUpsert.length) { + for (const { + deferred: { entity, locationKey }, + hash, + } of toUpsert) { + const entityRef = stringifyEntityRef(entity); + + try { + let ok = await this.updateUnprocessedEntity( + tx, + entity, + hash, + locationKey, + ); + if (!ok) { + ok = await this.insertUnprocessedEntity( + tx, + entity, + hash, + locationKey, + ); + } + + if (ok) { + await tx( + 'refresh_state_references', + ).insert({ + source_key: options.sourceKey, + target_entity_ref: entityRef, + }); + } else { + const conflictingKey = await this.checkLocationKeyConflict( + tx, + entityRef, + locationKey, + ); + if (conflictingKey) { + this.options.logger.warn( + `Source ${options.sourceKey} detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`, + ); + } + } + } catch (error) { + this.options.logger.error( + `Failed to add '${entityRef}' from source '${options.sourceKey}', ${error}`, + ); + } + } + } + } + + async refreshByRefreshKeys( + txOpaque: Transaction, + options: RefreshByKeyOptions, + ) { + const tx = txOpaque as Knex.Transaction; + const { keys } = options; + + await tx('refresh_state') + .whereIn('entity_id', function selectEntityRefs(tx2) { + tx2 + .whereIn('key', keys) + .select({ + entity_id: 'refresh_keys.entity_id', + }) + .from('refresh_keys'); + }) + .update({ next_update_at: tx.fn.now() }); + } + + /** + * Attempts to update an existing refresh state row, returning true if it was + * updated and false if there was no entity with a matching ref and location key. + * + * Updating the entity will also cause it to be scheduled for immediate processing. + */ + private async updateUnprocessedEntity( + tx: Knex.Transaction, + entity: Entity, + hash: string, + locationKey?: string, + ): Promise { + const entityRef = stringifyEntityRef(entity); + const serializedEntity = JSON.stringify(entity); + + const refreshResult = await tx('refresh_state') + .update({ + unprocessed_entity: serializedEntity, + unprocessed_hash: hash, + location_key: locationKey, + last_discovery_at: tx.fn.now(), + // We only get to this point if a processed entity actually had any changes, or + // if an entity provider requested this mutation, meaning that we can safely + // bump the deferred entities to the front of the queue for immediate processing. + next_update_at: tx.fn.now(), + }) + .where('entity_ref', entityRef) + .andWhere(inner => { + if (!locationKey) { + return inner.whereNull('location_key'); + } + return inner + .where('location_key', locationKey) + .orWhereNull('location_key'); + }); + + return refreshResult === 1; + } + + /** + * Attempts to insert a new refresh state row for the given entity, returning + * true if successful and false if there was a conflict. + */ + private async insertUnprocessedEntity( + tx: Knex.Transaction, + entity: Entity, + hash: string, + locationKey?: string, + ): Promise { + const entityRef = stringifyEntityRef(entity); + const serializedEntity = JSON.stringify(entity); + + try { + let query = tx('refresh_state').insert({ + entity_id: uuid(), + entity_ref: entityRef, + unprocessed_entity: serializedEntity, + unprocessed_hash: hash, + errors: '', + location_key: locationKey, + next_update_at: tx.fn.now(), + last_discovery_at: tx.fn.now(), + }); + + // TODO(Rugvip): only tested towards MySQL, Postgres and SQLite. + // We have to do this because the only way to detect if there was a conflict with + // SQLite is to catch the error, while Postgres needs to ignore the conflict to not + // break the ongoing transaction. + if (tx.client.config.client.includes('pg')) { + query = query.onConflict('entity_ref').ignore() as any; // type here does not match runtime + } + + // Postgres gives as an object with rowCount, SQLite gives us an array + const result: { rowCount?: number; length?: number } = await query; + return result.rowCount === 1 || result.length === 1; + } catch (error) { + // SQLite, or MySQL reached this rather than the rowCount check above + if (!isDatabaseConflictError(error)) { + throw error; + } else { + this.options.logger.debug( + `Unable to insert a new refresh state row, ${error}`, + ); + return false; + } + } + } + + /** + * Checks whether a refresh state exists for the given entity that has a + * location key that does not match the provided location key. + * + * @returns The conflicting key if there is one. + */ + private async checkLocationKeyConflict( + tx: Knex.Transaction, + entityRef: string, + locationKey?: string, + ): Promise { + const row = await tx('refresh_state') + .select('location_key') + .where('entity_ref', entityRef) + .first(); + + const conflictingKey = row?.location_key; + + // If there's no existing key we can't have a conflict + if (!conflictingKey) { + return undefined; + } + + if (conflictingKey !== locationKey) { + return conflictingKey; + } + return undefined; + } + + private async createDelta( + tx: Knex.Transaction, + options: ReplaceUnprocessedEntitiesOptions, + ): Promise<{ + toAdd: { deferred: DeferredEntity; hash: string }[]; + toUpsert: { deferred: DeferredEntity; hash: string }[]; + toRemove: string[]; + }> { + if (options.type === 'delta') { + return { + toAdd: [], + toUpsert: options.added.map(e => ({ + deferred: e, + hash: generateStableHash(e.entity), + })), + toRemove: options.removed.map(e => e.entityRef), + }; + } + + // Grab all of the existing references from the same source, and their locationKeys as well + const oldRefs = await tx( + 'refresh_state_references', + ) + .leftJoin('refresh_state', { + target_entity_ref: 'entity_ref', + }) + .where({ source_key: options.sourceKey }) + .select({ + target_entity_ref: 'refresh_state_references.target_entity_ref', + location_key: 'refresh_state.location_key', + unprocessed_hash: 'refresh_state.unprocessed_hash', + }); + + const items = options.items.map(deferred => ({ + deferred, + ref: stringifyEntityRef(deferred.entity), + hash: generateStableHash(deferred.entity), + })); + + const oldRefsSet = new Map( + oldRefs.map(r => [ + r.target_entity_ref, + { + locationKey: r.location_key, + oldEntityHash: r.unprocessed_hash, + }, + ]), + ); + const newRefsSet = new Set(items.map(item => item.ref)); + + const toAdd = new Array<{ deferred: DeferredEntity; hash: string }>(); + const toUpsert = new Array<{ deferred: DeferredEntity; hash: string }>(); + const toRemove = oldRefs + .map(row => row.target_entity_ref) + .filter(ref => !newRefsSet.has(ref)); + + for (const item of items) { + const oldRef = oldRefsSet.get(item.ref); + const upsertItem = { deferred: item.deferred, hash: item.hash }; + if (!oldRef) { + // Add any entity that does not exist in the database + toAdd.push(upsertItem); + } else if ( + (oldRef?.locationKey ?? undefined) !== + (item.deferred.locationKey ?? undefined) + ) { + // Remove and then re-add any entity that exists, but with a different location key + toRemove.push(item.ref); + toAdd.push(upsertItem); + } else if (oldRef.oldEntityHash !== item.hash) { + // Entities with modifications should be pushed through too + toUpsert.push(upsertItem); + } + } + + return { toAdd, toUpsert, toRemove }; + } +} diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 345aee6ebd..56e5f48f14 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -109,17 +109,12 @@ export type ListParentsResult = { entityRefs: string[]; }; +/** + * The database abstraction layer for Entity Processor interactions. + */ export interface ProcessingDatabase { transaction(fn: (tx: Transaction) => Promise): Promise; - /** - * Add unprocessed entities to the front of the processing queue using a mutation. - */ - replaceUnprocessedEntities( - txOpaque: Transaction, - options: ReplaceUnprocessedEntitiesOptions, - ): Promise; - getProcessableEntities( txOpaque: Transaction, request: { processBatchSize: number }, @@ -152,10 +147,25 @@ export interface ProcessingDatabase { options: UpdateProcessedEntityErrorsOptions, ): Promise; + listParents( + txOpaque: Transaction, + options: ListParentsOptions, + ): Promise; +} + +/** + * The database abstraction layer for Entity Provider interactions. + */ +export interface ProviderDatabase { + transaction(fn: (tx: Transaction) => Promise): Promise; + /** - * Schedules a refresh of a given entityRef. + * Add unprocessed entities to the front of the processing queue using a mutation. */ - refresh(txOpaque: Transaction, options: RefreshOptions): Promise; + replaceUnprocessedEntities( + txOpaque: Transaction, + options: ReplaceUnprocessedEntitiesOptions, + ): Promise; /** * Schedules a refresh for every entity that has a matching set of refresh key stored for it. @@ -164,6 +174,14 @@ export interface ProcessingDatabase { txOpaque: Transaction, options: RefreshByKeyOptions, ): Promise; +} + +// TODO(Rugvip): This is only partial for now +/** + * The database abstraction layer for catalog access. + */ +export interface CatalogDatabase { + transaction(fn: (tx: Transaction) => Promise): Promise; /** * Lists all ancestors of a given entityRef. @@ -175,8 +193,8 @@ export interface ProcessingDatabase { options: ListAncestorsOptions, ): Promise; - listParents( - txOpaque: Transaction, - options: ListParentsOptions, - ): Promise; + /** + * Schedules a refresh of a given entityRef. + */ + refresh(txOpaque: Transaction, options: RefreshOptions): Promise; } diff --git a/plugins/catalog-backend/src/integration.test.ts b/plugins/catalog-backend/src/integration.test.ts index a63119bb67..4ba046834b 100644 --- a/plugins/catalog-backend/src/integration.test.ts +++ b/plugins/catalog-backend/src/integration.test.ts @@ -29,6 +29,7 @@ import { import { defaultEntityDataParser } from './modules/util/parse'; import { DefaultCatalogProcessingOrchestrator } from './processing/DefaultCatalogProcessingOrchestrator'; import { applyDatabaseMigrations } from './database/migrations'; +import { DefaultCatalogDatabase } from './database/DefaultCatalogDatabase'; import { DefaultProcessingDatabase } from './database/DefaultProcessingDatabase'; import { ScmIntegrations } from '@backstage/integration'; import { DefaultCatalogRulesEnforcer } from './ingestion/CatalogRules'; @@ -50,6 +51,7 @@ import { processingResult, } from '@backstage/plugin-catalog-node'; import { RefreshStateItem } from './database/types'; +import { DefaultProviderDatabase } from './database/DefaultProviderDatabase'; const voidLogger = getVoidLogger(); @@ -213,6 +215,14 @@ class TestHarness { await applyDatabaseMigrations(db); + const catalogDatabase = new DefaultCatalogDatabase({ + database: db, + logger, + }); + const providerDatabase = new DefaultProviderDatabase({ + database: db, + logger, + }); const processingDatabase = new DefaultProcessingDatabase({ database: db, logger, @@ -272,11 +282,11 @@ class TestHarness { proxyProgressTracker, ); - const refresh = new DefaultRefreshService({ database: processingDatabase }); + const refresh = new DefaultRefreshService({ database: catalogDatabase }); const provider = new TestProvider(); - await connectEntityProviders(processingDatabase, [provider]); + await connectEntityProviders(providerDatabase, [provider]); return new TestHarness( catalog, diff --git a/plugins/catalog-backend/src/processing/connectEntityProviders.ts b/plugins/catalog-backend/src/processing/connectEntityProviders.ts index 3eb638ce7a..30d30ca9b6 100644 --- a/plugins/catalog-backend/src/processing/connectEntityProviders.ts +++ b/plugins/catalog-backend/src/processing/connectEntityProviders.ts @@ -19,7 +19,7 @@ import { entityEnvelopeSchemaValidator, stringifyEntityRef, } from '@backstage/catalog-model'; -import { ProcessingDatabase } from '../database/types'; +import { ProviderDatabase } from '../database/types'; import { EntityProvider, EntityProviderConnection, @@ -33,12 +33,12 @@ class Connection implements EntityProviderConnection { constructor( private readonly config: { id: string; - processingDatabase: ProcessingDatabase; + providerDatabase: ProviderDatabase; }, ) {} async applyMutation(mutation: EntityProviderMutation): Promise { - const db = this.config.processingDatabase; + const db = this.config.providerDatabase; if (mutation.type === 'full') { this.check(mutation.entities.map(e => e.entity)); @@ -75,7 +75,7 @@ class Connection implements EntityProviderConnection { } async refresh(options: EntityProviderRefreshOptions): Promise { - const db = this.config.processingDatabase; + const db = this.config.providerDatabase; await db.transaction(async (tx: any) => { return db.refreshByRefreshKeys(tx, { @@ -96,14 +96,14 @@ class Connection implements EntityProviderConnection { } export async function connectEntityProviders( - db: ProcessingDatabase, + db: ProviderDatabase, providers: EntityProvider[], ) { await Promise.all( providers.map(async provider => { const connection = new Connection({ id: provider.getProviderName(), - processingDatabase: db, + providerDatabase: db, }); return provider.connect(connection); }), diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index ee5e5c48f2..c0d0e5db0d 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -29,6 +29,7 @@ export type EntityProcessingRequest = { entity: Entity; state?: JsonObject; // Versions for multiple deployments etc }; + /** * The result of processing an entity. * @internal @@ -57,13 +58,18 @@ export type RefreshKeyData = { /** * Responsible for executing the individual processing steps in order to fully process an entity. - * @public */ export interface CatalogProcessingOrchestrator { process(request: EntityProcessingRequest): Promise; } -/** @public */ +/** + * Represents the engine that drives the processing loops. Some backend + * instances may choose to not call start, if they focus only on API + * interactions. + * + * @public + */ export interface CatalogProcessingEngine { start(): Promise; stop(): Promise; diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts index cbaf876071..829e41c21e 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts @@ -28,6 +28,8 @@ describe('AuthorizedEntitiesCatalog', () => { removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), + refresh: jest.fn(), + listAncestors: jest.fn(), }; const fakePermissionApi = { authorize: jest.fn(), diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 8aab74c4ba..48bf6a08ef 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -96,6 +96,8 @@ import { RESOURCE_TYPE_CATALOG_ENTITY, } from '@backstage/plugin-catalog-common'; import { AuthorizedLocationService } from './AuthorizedLocationService'; +import { DefaultProviderDatabase } from '../database/DefaultProviderDatabase'; +import { DefaultCatalogDatabase } from '../database/DefaultCatalogDatabase'; /** @public */ export type CatalogEnvironment = { @@ -431,6 +433,14 @@ export class CatalogBuilder { logger, refreshInterval: this.processingInterval, }); + const providerDatabase = new DefaultProviderDatabase({ + database: dbClient, + logger, + }); + const catalogDatabase = new DefaultCatalogDatabase({ + database: dbClient, + logger, + }); const integrations = ScmIntegrations.fromConfig(config); const rulesEnforcer = DefaultCatalogRulesEnforcer.fromConfig(config); const orchestrator = new DefaultCatalogProcessingOrchestrator({ @@ -520,7 +530,7 @@ export class CatalogBuilder { permissionEvaluator, ); const refreshService = new AuthorizedRefreshService( - new DefaultRefreshService({ database: processingDatabase }), + new DefaultRefreshService({ database: catalogDatabase }), permissionEvaluator, ); const router = await createRouter({ @@ -534,7 +544,7 @@ export class CatalogBuilder { permissionIntegrationRouter, }); - await connectEntityProviders(processingDatabase, entityProviders); + await connectEntityProviders(providerDatabase, entityProviders); return { processingEngine, diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts index 3ed4f3fe36..eafa0b0c14 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.test.ts @@ -16,11 +16,14 @@ import { getVoidLogger } from '@backstage/backend-common'; import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; +import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { createHash } from 'crypto'; import { Knex } from 'knex'; +import { v4 as uuid } from 'uuid'; import { Logger } from 'winston'; -import { applyDatabaseMigrations } from '../database/migrations'; +import { DefaultCatalogDatabase } from '../database/DefaultCatalogDatabase'; import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; +import { applyDatabaseMigrations } from '../database/migrations'; import { DbRefreshStateReferencesRow, DbRefreshStateRow, @@ -29,11 +32,9 @@ import { ProcessingDatabase } from '../database/types'; import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine'; import { EntityProcessingRequest } from '../processing/types'; import { Stitcher } from '../stitching/Stitcher'; -import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; -import { v4 as uuid } from 'uuid'; import { DefaultRefreshService } from './DefaultRefreshService'; -describe('Refresh integration', () => { +describe('DefaultRefreshService', () => { const defaultLogger = getVoidLogger(); const databases = TestDatabases.create({ ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], @@ -47,11 +48,15 @@ describe('Refresh integration', () => { await applyDatabaseMigrations(knex); return { knex, - db: new DefaultProcessingDatabase({ + processingDb: new DefaultProcessingDatabase({ database: knex, logger, refreshInterval: () => 100, }), + catalogDb: new DefaultCatalogDatabase({ + database: knex, + logger, + }), }; } @@ -176,10 +181,12 @@ describe('Refresh integration', () => { it.each(databases.eachSupportedId())( 'should refresh the parent location, %p', async databaseId => { - const { knex, db } = await createDatabase(databaseId); - const refreshService = new DefaultRefreshService({ database: db }); + const { knex, processingDb, catalogDb } = await createDatabase( + databaseId, + ); + const refreshService = new DefaultRefreshService({ database: catalogDb }); const engine = await createPopulatedEngine({ - db, + db: processingDb, knex, entities: [ { @@ -220,10 +227,12 @@ describe('Refresh integration', () => { it.each(databases.eachSupportedId())( 'should refresh the location further up the tree, %p', async databaseId => { - const { knex, db } = await createDatabase(databaseId); - const refreshService = new DefaultRefreshService({ database: db }); + const { knex, processingDb, catalogDb } = await createDatabase( + databaseId, + ); + const refreshService = new DefaultRefreshService({ database: catalogDb }); const engine = await createPopulatedEngine({ - db, + db: processingDb, knex, entities: [ { @@ -273,10 +282,12 @@ describe('Refresh integration', () => { 'should refresh even when parent has no changes', async databaseId => { let secondRound = false; - const { knex, db } = await createDatabase(databaseId); - const refreshService = new DefaultRefreshService({ database: db }); + const { knex, processingDb, catalogDb } = await createDatabase( + databaseId, + ); + const refreshService = new DefaultRefreshService({ database: catalogDb }); const engine = await createPopulatedEngine({ - db, + db: processingDb, knex, entities: [ { diff --git a/plugins/catalog-backend/src/service/DefaultRefreshService.ts b/plugins/catalog-backend/src/service/DefaultRefreshService.ts index 3b982a0e46..99b78f900b 100644 --- a/plugins/catalog-backend/src/service/DefaultRefreshService.ts +++ b/plugins/catalog-backend/src/service/DefaultRefreshService.ts @@ -14,13 +14,13 @@ * limitations under the License. */ -import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; +import { DefaultCatalogDatabase } from '../database/DefaultCatalogDatabase'; import { RefreshOptions, RefreshService } from './types'; export class DefaultRefreshService implements RefreshService { - private database: DefaultProcessingDatabase; + private database: DefaultCatalogDatabase; - constructor(options: { database: DefaultProcessingDatabase }) { + constructor(options: { database: DefaultCatalogDatabase }) { this.database = options.database; } From effc37cf492deb7bb92c4135ff56754161927644 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 9 Dec 2022 14:29:45 +0100 Subject: [PATCH 126/437] catalog-backend: split out some common DB operations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: Patrik Oldsberg --- .../src/database/DefaultProcessingDatabase.ts | 151 +++--------------- .../src/database/DefaultProviderDatabase.ts | 134 +--------------- .../refreshState/checkLocationKeyConflict.ts | 47 ++++++ .../refreshState/insertUnprocessedEntity.ts | 70 ++++++++ .../refreshState/updateUnprocessedEntity.ts | 58 +++++++ 5 files changed, 201 insertions(+), 259 deletions(-) create mode 100644 plugins/catalog-backend/src/database/operations/refreshState/checkLocationKeyConflict.ts create mode 100644 plugins/catalog-backend/src/database/operations/refreshState/insertUnprocessedEntity.ts create mode 100644 plugins/catalog-backend/src/database/operations/refreshState/updateUnprocessedEntity.ts diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index c32f7d6f3d..6e77a6a0d8 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -18,18 +18,7 @@ import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { ConflictError } from '@backstage/errors'; import { Knex } from 'knex'; import lodash from 'lodash'; -import { v4 as uuid } from 'uuid'; import type { Logger } from 'winston'; -import { - Transaction, - GetProcessableEntitiesResult, - ProcessingDatabase, - RefreshStateItem, - UpdateProcessedEntityOptions, - UpdateEntityCacheOptions, - ListParentsOptions, - ListParentsResult, -} from './types'; import { ProcessingIntervalFunction } from '../processing/refresh'; import { rethrowError, timestampToDateTime } from './conversion'; import { initDatabaseMetrics } from './metrics'; @@ -39,10 +28,22 @@ import { DbRefreshStateRow, DbRelationsRow, } from './tables'; +import { + GetProcessableEntitiesResult, + ListParentsOptions, + ListParentsResult, + ProcessingDatabase, + RefreshStateItem, + Transaction, + UpdateEntityCacheOptions, + UpdateProcessedEntityOptions, +} from './types'; -import { generateStableHash } from './util'; -import { isDatabaseConflictError } from '@backstage/backend-common'; import { DeferredEntity } from '@backstage/plugin-catalog-node'; +import { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict'; +import { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity'; +import { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity'; +import { generateStableHash } from './util'; // The number of items that are sent per batch to the database layer, when // doing .batchInsert calls to knex. This needs to be low enough to not cause @@ -291,123 +292,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { } } - /** - * Attempts to update an existing refresh state row, returning true if it was - * updated and false if there was no entity with a matching ref and location key. - * - * Updating the entity will also cause it to be scheduled for immediate processing. - */ - private async updateUnprocessedEntity( - tx: Knex.Transaction, - entity: Entity, - hash: string, - locationKey?: string, - ): Promise { - const entityRef = stringifyEntityRef(entity); - const serializedEntity = JSON.stringify(entity); - - const refreshResult = await tx('refresh_state') - .update({ - unprocessed_entity: serializedEntity, - unprocessed_hash: hash, - location_key: locationKey, - last_discovery_at: tx.fn.now(), - // We only get to this point if a processed entity actually had any changes, or - // if an entity provider requested this mutation, meaning that we can safely - // bump the deferred entities to the front of the queue for immediate processing. - next_update_at: tx.fn.now(), - }) - .where('entity_ref', entityRef) - .andWhere(inner => { - if (!locationKey) { - return inner.whereNull('location_key'); - } - return inner - .where('location_key', locationKey) - .orWhereNull('location_key'); - }); - - return refreshResult === 1; - } - - /** - * Attempts to insert a new refresh state row for the given entity, returning - * true if successful and false if there was a conflict. - */ - private async insertUnprocessedEntity( - tx: Knex.Transaction, - entity: Entity, - hash: string, - locationKey?: string, - ): Promise { - const entityRef = stringifyEntityRef(entity); - const serializedEntity = JSON.stringify(entity); - - try { - let query = tx('refresh_state').insert({ - entity_id: uuid(), - entity_ref: entityRef, - unprocessed_entity: serializedEntity, - unprocessed_hash: hash, - errors: '', - location_key: locationKey, - next_update_at: tx.fn.now(), - last_discovery_at: tx.fn.now(), - }); - - // TODO(Rugvip): only tested towards MySQL, Postgres and SQLite. - // We have to do this because the only way to detect if there was a conflict with - // SQLite is to catch the error, while Postgres needs to ignore the conflict to not - // break the ongoing transaction. - if (tx.client.config.client.includes('pg')) { - query = query.onConflict('entity_ref').ignore() as any; // type here does not match runtime - } - - // Postgres gives as an object with rowCount, SQLite gives us an array - const result: { rowCount?: number; length?: number } = await query; - return result.rowCount === 1 || result.length === 1; - } catch (error) { - // SQLite, or MySQL reached this rather than the rowCount check above - if (!isDatabaseConflictError(error)) { - throw error; - } else { - this.options.logger.debug( - `Unable to insert a new refresh state row, ${error}`, - ); - return false; - } - } - } - - /** - * Checks whether a refresh state exists for the given entity that has a - * location key that does not match the provided location key. - * - * @returns The conflicting key if there is one. - */ - private async checkLocationKeyConflict( - tx: Knex.Transaction, - entityRef: string, - locationKey?: string, - ): Promise { - const row = await tx('refresh_state') - .select('location_key') - .where('entity_ref', entityRef) - .first(); - - const conflictingKey = row?.location_key; - - // If there's no existing key we can't have a conflict - if (!conflictingKey) { - return undefined; - } - - if (conflictingKey !== locationKey) { - return conflictingKey; - } - return undefined; - } - private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] { return lodash.uniqBy( rows, @@ -438,7 +322,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const entityRef = stringifyEntityRef(entity); const hash = generateStableHash(entity); - const updated = await this.updateUnprocessedEntity( + const updated = await updateUnprocessedEntity( tx, entity, hash, @@ -449,10 +333,11 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { continue; } - const inserted = await this.insertUnprocessedEntity( + const inserted = await insertUnprocessedEntity( tx, entity, hash, + this.options.logger, locationKey, ); if (inserted) { @@ -463,7 +348,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { // If the row can't be inserted, we have a conflict, but it could be either // because of a conflicting locationKey or a race with another instance, so check // whether the conflicting entity has the same entityRef but a different locationKey - const conflictingKey = await this.checkLocationKeyConflict( + const conflictingKey = await checkLocationKeyConflict( tx, entityRef, locationKey, diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts index 05a1011c8d..9dbdc5f516 100644 --- a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts @@ -15,13 +15,16 @@ */ import { isDatabaseConflictError } from '@backstage/backend-common'; -import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; +import { stringifyEntityRef } from '@backstage/catalog-model'; import { DeferredEntity } from '@backstage/plugin-catalog-node'; import { Knex } from 'knex'; import lodash from 'lodash'; import { v4 as uuid } from 'uuid'; import type { Logger } from 'winston'; import { rethrowError } from './conversion'; +import { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict'; +import { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity'; +import { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity'; import { DbRefreshStateReferencesRow, DbRefreshStateRow } from './tables'; import { ProviderDatabase, @@ -255,17 +258,13 @@ export class DefaultProviderDatabase implements ProviderDatabase { const entityRef = stringifyEntityRef(entity); try { - let ok = await this.updateUnprocessedEntity( - tx, - entity, - hash, - locationKey, - ); + let ok = await updateUnprocessedEntity(tx, entity, hash, locationKey); if (!ok) { - ok = await this.insertUnprocessedEntity( + ok = await insertUnprocessedEntity( tx, entity, hash, + this.options.logger, locationKey, ); } @@ -278,7 +277,7 @@ export class DefaultProviderDatabase implements ProviderDatabase { target_entity_ref: entityRef, }); } else { - const conflictingKey = await this.checkLocationKeyConflict( + const conflictingKey = await checkLocationKeyConflict( tx, entityRef, locationKey, @@ -317,123 +316,6 @@ export class DefaultProviderDatabase implements ProviderDatabase { .update({ next_update_at: tx.fn.now() }); } - /** - * Attempts to update an existing refresh state row, returning true if it was - * updated and false if there was no entity with a matching ref and location key. - * - * Updating the entity will also cause it to be scheduled for immediate processing. - */ - private async updateUnprocessedEntity( - tx: Knex.Transaction, - entity: Entity, - hash: string, - locationKey?: string, - ): Promise { - const entityRef = stringifyEntityRef(entity); - const serializedEntity = JSON.stringify(entity); - - const refreshResult = await tx('refresh_state') - .update({ - unprocessed_entity: serializedEntity, - unprocessed_hash: hash, - location_key: locationKey, - last_discovery_at: tx.fn.now(), - // We only get to this point if a processed entity actually had any changes, or - // if an entity provider requested this mutation, meaning that we can safely - // bump the deferred entities to the front of the queue for immediate processing. - next_update_at: tx.fn.now(), - }) - .where('entity_ref', entityRef) - .andWhere(inner => { - if (!locationKey) { - return inner.whereNull('location_key'); - } - return inner - .where('location_key', locationKey) - .orWhereNull('location_key'); - }); - - return refreshResult === 1; - } - - /** - * Attempts to insert a new refresh state row for the given entity, returning - * true if successful and false if there was a conflict. - */ - private async insertUnprocessedEntity( - tx: Knex.Transaction, - entity: Entity, - hash: string, - locationKey?: string, - ): Promise { - const entityRef = stringifyEntityRef(entity); - const serializedEntity = JSON.stringify(entity); - - try { - let query = tx('refresh_state').insert({ - entity_id: uuid(), - entity_ref: entityRef, - unprocessed_entity: serializedEntity, - unprocessed_hash: hash, - errors: '', - location_key: locationKey, - next_update_at: tx.fn.now(), - last_discovery_at: tx.fn.now(), - }); - - // TODO(Rugvip): only tested towards MySQL, Postgres and SQLite. - // We have to do this because the only way to detect if there was a conflict with - // SQLite is to catch the error, while Postgres needs to ignore the conflict to not - // break the ongoing transaction. - if (tx.client.config.client.includes('pg')) { - query = query.onConflict('entity_ref').ignore() as any; // type here does not match runtime - } - - // Postgres gives as an object with rowCount, SQLite gives us an array - const result: { rowCount?: number; length?: number } = await query; - return result.rowCount === 1 || result.length === 1; - } catch (error) { - // SQLite, or MySQL reached this rather than the rowCount check above - if (!isDatabaseConflictError(error)) { - throw error; - } else { - this.options.logger.debug( - `Unable to insert a new refresh state row, ${error}`, - ); - return false; - } - } - } - - /** - * Checks whether a refresh state exists for the given entity that has a - * location key that does not match the provided location key. - * - * @returns The conflicting key if there is one. - */ - private async checkLocationKeyConflict( - tx: Knex.Transaction, - entityRef: string, - locationKey?: string, - ): Promise { - const row = await tx('refresh_state') - .select('location_key') - .where('entity_ref', entityRef) - .first(); - - const conflictingKey = row?.location_key; - - // If there's no existing key we can't have a conflict - if (!conflictingKey) { - return undefined; - } - - if (conflictingKey !== locationKey) { - return conflictingKey; - } - return undefined; - } - private async createDelta( tx: Knex.Transaction, options: ReplaceUnprocessedEntitiesOptions, diff --git a/plugins/catalog-backend/src/database/operations/refreshState/checkLocationKeyConflict.ts b/plugins/catalog-backend/src/database/operations/refreshState/checkLocationKeyConflict.ts new file mode 100644 index 0000000000..a256b36729 --- /dev/null +++ b/plugins/catalog-backend/src/database/operations/refreshState/checkLocationKeyConflict.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; +import { DbRefreshStateRow } from '../../tables'; + +/** + * Checks whether a refresh state exists for the given entity that has a + * location key that does not match the provided location key. + * + * @returns The conflicting key if there is one. + */ +export async function checkLocationKeyConflict( + tx: Knex.Transaction, + entityRef: string, + locationKey?: string, +): Promise { + const row = await tx('refresh_state') + .select('location_key') + .where('entity_ref', entityRef) + .first(); + + const conflictingKey = row?.location_key; + + // If there's no existing key we can't have a conflict + if (!conflictingKey) { + return undefined; + } + + if (conflictingKey !== locationKey) { + return conflictingKey; + } + return undefined; +} diff --git a/plugins/catalog-backend/src/database/operations/refreshState/insertUnprocessedEntity.ts b/plugins/catalog-backend/src/database/operations/refreshState/insertUnprocessedEntity.ts new file mode 100644 index 0000000000..bf34fd2ce8 --- /dev/null +++ b/plugins/catalog-backend/src/database/operations/refreshState/insertUnprocessedEntity.ts @@ -0,0 +1,70 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; +import { Knex } from 'knex'; +import { DbRefreshStateRow } from '../../tables'; +import { v4 as uuid } from 'uuid'; +import type { Logger } from 'winston'; +import { isDatabaseConflictError } from '@backstage/backend-common'; + +/** + * Attempts to insert a new refresh state row for the given entity, returning + * true if successful and false if there was a conflict. + */ +export async function insertUnprocessedEntity( + tx: Knex.Transaction, + entity: Entity, + hash: string, + logger: Logger, + locationKey?: string, +): Promise { + const entityRef = stringifyEntityRef(entity); + const serializedEntity = JSON.stringify(entity); + + try { + let query = tx('refresh_state').insert({ + entity_id: uuid(), + entity_ref: entityRef, + unprocessed_entity: serializedEntity, + unprocessed_hash: hash, + errors: '', + location_key: locationKey, + next_update_at: tx.fn.now(), + last_discovery_at: tx.fn.now(), + }); + + // TODO(Rugvip): only tested towards MySQL, Postgres and SQLite. + // We have to do this because the only way to detect if there was a conflict with + // SQLite is to catch the error, while Postgres needs to ignore the conflict to not + // break the ongoing transaction. + if (tx.client.config.client.includes('pg')) { + query = query.onConflict('entity_ref').ignore() as any; // type here does not match runtime + } + + // Postgres gives as an object with rowCount, SQLite gives us an array + const result: { rowCount?: number; length?: number } = await query; + return result.rowCount === 1 || result.length === 1; + } catch (error) { + // SQLite, or MySQL reached this rather than the rowCount check above + if (!isDatabaseConflictError(error)) { + throw error; + } else { + logger.debug(`Unable to insert a new refresh state row, ${error}`); + return false; + } + } +} diff --git a/plugins/catalog-backend/src/database/operations/refreshState/updateUnprocessedEntity.ts b/plugins/catalog-backend/src/database/operations/refreshState/updateUnprocessedEntity.ts new file mode 100644 index 0000000000..084454de66 --- /dev/null +++ b/plugins/catalog-backend/src/database/operations/refreshState/updateUnprocessedEntity.ts @@ -0,0 +1,58 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; +import { Knex } from 'knex'; +import { DbRefreshStateRow } from '../../tables'; + +/** + * Attempts to update an existing refresh state row, returning true if it was + * updated and false if there was no entity with a matching ref and location key. + * + * Updating the entity will also cause it to be scheduled for immediate processing. + */ +export async function updateUnprocessedEntity( + tx: Knex.Transaction, + entity: Entity, + hash: string, + locationKey?: string, +): Promise { + const entityRef = stringifyEntityRef(entity); + const serializedEntity = JSON.stringify(entity); + + const refreshResult = await tx('refresh_state') + .update({ + unprocessed_entity: serializedEntity, + unprocessed_hash: hash, + location_key: locationKey, + last_discovery_at: tx.fn.now(), + // We only get to this point if a processed entity actually had any changes, or + // if an entity provider requested this mutation, meaning that we can safely + // bump the deferred entities to the front of the queue for immediate processing. + next_update_at: tx.fn.now(), + }) + .where('entity_ref', entityRef) + .andWhere(inner => { + if (!locationKey) { + return inner.whereNull('location_key'); + } + return inner + .where('location_key', locationKey) + .orWhereNull('location_key'); + }); + + return refreshResult === 1; +} From a162f33a5f9ca17071b9418b14ed3e36dbfadbcb Mon Sep 17 00:00:00 2001 From: Djam Date: Fri, 9 Dec 2022 15:32:02 +0100 Subject: [PATCH 127/437] Update automate_changeset_feedback.yml Signed-off-by: Djam --- .github/workflows/automate_changeset_feedback.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/automate_changeset_feedback.yml b/.github/workflows/automate_changeset_feedback.yml index b6b5ea3d7e..3637ed11e7 100644 --- a/.github/workflows/automate_changeset_feedback.yml +++ b/.github/workflows/automate_changeset_feedback.yml @@ -21,10 +21,13 @@ jobs: if: github.repository == 'backstage/backstage' && github.event.pull_request.user.login != 'backstage-service' runs-on: ubuntu-latest steps: - - uses: backstage/actions/changeset-feedback@v0.5.9 + - uses: backstage/actions/changeset-feedback@vchangeset-patches with: # Fetch the commit that's merged into the base rather than the target ref # This will let us diff only the contents of the PR, without fetching more history diffRef: 'refs/pull/${{ github.event.pull_request.number }}/merge' github-token: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }} issue-number: ${{ steps.pr-number.outputs.pr-number }} + app-id: ${{ secrets.BACKSTAGE_GOALIE_APPLICATION_ID }} + private-key: ${{ secrets.BACKSTAGE_GOALIE_PRIVATE_KEY }} + installation-id: ${{ secrets.BACKSTAGE_GOALIE_INSTALLATION_ID }} From f9f4f87a78f0b4ab6d6d0a4080f9cc00dda00ef8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 9 Dec 2022 16:07:26 +0100 Subject: [PATCH 128/437] catalog-backend: split out pruned delete and refresh by key operations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: Patrik Oldsberg --- .../src/database/DefaultProcessingDatabase.ts | 14 +- .../src/database/DefaultProviderDatabase.ts | 161 +++--------------- .../deleteWithEagerPruningOfChildren.ts | 155 +++++++++++++++++ .../provider/refreshByRefreshKeys.ts | 43 +++++ .../refreshState/checkLocationKeyConflict.ts | 12 +- .../refreshState/insertUnprocessedEntity.ts | 16 +- .../refreshState/updateUnprocessedEntity.ts | 14 +- 7 files changed, 250 insertions(+), 165 deletions(-) create mode 100644 plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.ts create mode 100644 plugins/catalog-backend/src/database/operations/provider/refreshByRefreshKeys.ts diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 6e77a6a0d8..1127deed7e 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -322,24 +322,24 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const entityRef = stringifyEntityRef(entity); const hash = generateStableHash(entity); - const updated = await updateUnprocessedEntity( + const updated = await updateUnprocessedEntity({ tx, entity, hash, locationKey, - ); + }); if (updated) { stateReferences.push(entityRef); continue; } - const inserted = await insertUnprocessedEntity( + const inserted = await insertUnprocessedEntity({ tx, entity, hash, - this.options.logger, locationKey, - ); + logger: this.options.logger, + }); if (inserted) { stateReferences.push(entityRef); continue; @@ -348,11 +348,11 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { // If the row can't be inserted, we have a conflict, but it could be either // because of a conflicting locationKey or a race with another instance, so check // whether the conflicting entity has the same entityRef but a different locationKey - const conflictingKey = await checkLocationKeyConflict( + const conflictingKey = await checkLocationKeyConflict({ tx, entityRef, locationKey, - ); + }); if (conflictingKey) { this.options.logger.warn( `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`, diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts index 9dbdc5f516..f1073fd984 100644 --- a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts @@ -22,6 +22,8 @@ import lodash from 'lodash'; import { v4 as uuid } from 'uuid'; import type { Logger } from 'winston'; import { rethrowError } from './conversion'; +import { deleteWithEagerPruningOfChildren } from './operations/provider/deleteWithEagerPruningOfChildren'; +import { refreshByRefreshKeys } from './operations/provider/refreshByRefreshKeys'; import { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict'; import { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity'; import { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity'; @@ -51,10 +53,10 @@ export class DefaultProviderDatabase implements ProviderDatabase { async transaction(fn: (tx: Transaction) => Promise): Promise { try { let result: T | undefined = undefined; - await this.options.database.transaction( async tx => { - // We can't return here, as knex swallows the return type in case the transaction is rolled back: + // We can't return here, as knex swallows the return type in case the + // transaction is rolled back: // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136 result = await fn(tx); }, @@ -63,7 +65,6 @@ export class DefaultProviderDatabase implements ProviderDatabase { doNotRejectOnRollback: true, }, ); - return result!; } catch (e) { this.options.logger.debug(`Error during transaction, ${e}`); @@ -76,128 +77,14 @@ export class DefaultProviderDatabase implements ProviderDatabase { options: ReplaceUnprocessedEntitiesOptions, ): Promise { const tx = txOpaque as Knex.Transaction; - const { toAdd, toUpsert, toRemove } = await this.createDelta(tx, options); if (toRemove.length) { - let removedCount = 0; - const rootId = () => { - if (tx.client.config.client.includes('mysql')) { - return tx.raw('CAST(NULL as UNSIGNED INT)', []); - } - - return tx.raw('CAST(NULL as INT)', []); - }; - for (const refs of lodash.chunk(toRemove, 1000)) { - /* - WITH RECURSIVE - -- All the nodes that can be reached downwards from our root - descendants(root_id, entity_ref) AS ( - SELECT id, target_entity_ref - FROM refresh_state_references - WHERE source_key = "R1" AND target_entity_ref = "A" - UNION - SELECT descendants.root_id, target_entity_ref - FROM descendants - JOIN refresh_state_references ON source_entity_ref = descendants.entity_ref - ), - -- All the nodes that can be reached upwards from the descendants - ancestors(root_id, via_entity_ref, to_entity_ref) AS ( - SELECT CAST(NULL as INT), entity_ref, entity_ref - FROM descendants - UNION - SELECT - CASE WHEN source_key IS NOT NULL THEN id ELSE NULL END, - source_entity_ref, - ancestors.to_entity_ref - FROM ancestors - JOIN refresh_state_references ON target_entity_ref = ancestors.via_entity_ref - ) - -- Start out with all of the descendants - SELECT descendants.entity_ref - FROM descendants - -- Expand with all ancestors that point to those, but aren't the current root - LEFT OUTER JOIN ancestors - ON ancestors.to_entity_ref = descendants.entity_ref - AND ancestors.root_id IS NOT NULL - AND ancestors.root_id != descendants.root_id - -- Exclude all lines that had such a foreign ancestor - WHERE ancestors.root_id IS NULL; - */ - removedCount += await tx('refresh_state') - .whereIn('entity_ref', function orphanedEntityRefs(orphans) { - return ( - orphans - // All the nodes that can be reached downwards from our root - .withRecursive('descendants', function descendants(outer) { - return outer - .select({ root_id: 'id', entity_ref: 'target_entity_ref' }) - .from('refresh_state_references') - .where('source_key', options.sourceKey) - .whereIn('target_entity_ref', refs) - .union(function recursive(inner) { - return inner - .select({ - root_id: 'descendants.root_id', - entity_ref: - 'refresh_state_references.target_entity_ref', - }) - .from('descendants') - .join('refresh_state_references', { - 'descendants.entity_ref': - 'refresh_state_references.source_entity_ref', - }); - }); - }) - // All the nodes that can be reached upwards from the descendants - .withRecursive('ancestors', function ancestors(outer) { - return outer - .select({ - root_id: rootId(), - via_entity_ref: 'entity_ref', - to_entity_ref: 'entity_ref', - }) - .from('descendants') - .union(function recursive(inner) { - return inner - .select({ - root_id: tx.raw( - 'CASE WHEN source_key IS NOT NULL THEN id ELSE NULL END', - [], - ), - via_entity_ref: 'source_entity_ref', - to_entity_ref: 'ancestors.to_entity_ref', - }) - .from('ancestors') - .join('refresh_state_references', { - target_entity_ref: 'ancestors.via_entity_ref', - }); - }); - }) - // Start out with all of the descendants - .select('descendants.entity_ref') - .from('descendants') - // Expand with all ancestors that point to those, but aren't the current root - .leftOuterJoin('ancestors', function keepaliveRoots() { - this.on( - 'ancestors.to_entity_ref', - '=', - 'descendants.entity_ref', - ); - this.andOnNotNull('ancestors.root_id'); - this.andOn('ancestors.root_id', '!=', 'descendants.root_id'); - }) - .whereNull('ancestors.root_id') - ); - }) - .delete(); - - await tx('refresh_state_references') - .where('source_key', '=', options.sourceKey) - .whereIn('target_entity_ref', refs) - .delete(); - } - + const removedCount = await deleteWithEagerPruningOfChildren({ + tx, + entityRefs: toRemove, + sourceKey: options.sourceKey, + }); this.options.logger.debug( `removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`, ); @@ -258,15 +145,20 @@ export class DefaultProviderDatabase implements ProviderDatabase { const entityRef = stringifyEntityRef(entity); try { - let ok = await updateUnprocessedEntity(tx, entity, hash, locationKey); + let ok = await updateUnprocessedEntity({ + tx, + entity, + hash, + locationKey, + }); if (!ok) { - ok = await insertUnprocessedEntity( + ok = await insertUnprocessedEntity({ tx, entity, hash, - this.options.logger, locationKey, - ); + logger: this.options.logger, + }); } if (ok) { @@ -277,11 +169,11 @@ export class DefaultProviderDatabase implements ProviderDatabase { target_entity_ref: entityRef, }); } else { - const conflictingKey = await checkLocationKeyConflict( + const conflictingKey = await checkLocationKeyConflict({ tx, entityRef, locationKey, - ); + }); if (conflictingKey) { this.options.logger.warn( `Source ${options.sourceKey} detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`, @@ -302,18 +194,7 @@ export class DefaultProviderDatabase implements ProviderDatabase { options: RefreshByKeyOptions, ) { const tx = txOpaque as Knex.Transaction; - const { keys } = options; - - await tx('refresh_state') - .whereIn('entity_id', function selectEntityRefs(tx2) { - tx2 - .whereIn('key', keys) - .select({ - entity_id: 'refresh_keys.entity_id', - }) - .from('refresh_keys'); - }) - .update({ next_update_at: tx.fn.now() }); + await refreshByRefreshKeys({ tx, keys: options.keys }); } private async createDelta( diff --git a/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.ts b/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.ts new file mode 100644 index 0000000000..5e2aa482a3 --- /dev/null +++ b/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.ts @@ -0,0 +1,155 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; +import lodash from 'lodash'; +import { DbRefreshStateReferencesRow, DbRefreshStateRow } from '../../tables'; + +/** + * Given a number of entity refs originally created by a given entity provider + * (source key), remove those entities from the refresh state, and at the same + * time recursively remove every child that is a direct or indirect result of + * processing those entities, if they would have otherwise become orphaned by + * the removal of their parents. + */ +export async function deleteWithEagerPruningOfChildren(options: { + tx: Knex.Transaction; + entityRefs: string[]; + sourceKey: string; +}): Promise { + const { tx, entityRefs, sourceKey } = options; + let removedCount = 0; + + const rootId = () => + tx.raw( + tx.client.config.client.includes('mysql') + ? 'CAST(NULL as UNSIGNED INT)' + : 'CAST(NULL as UNSIGNED INT)', + [], + ); + + // Split up the operation by (large) chunks, so that we do not hit database + // limits for the number of permitted bindings on a precompiled statement + for (const refs of lodash.chunk(entityRefs, 1000)) { + /* + WITH RECURSIVE + -- All the nodes that can be reached downwards from our root + descendants(root_id, entity_ref) AS ( + SELECT id, target_entity_ref + FROM refresh_state_references + WHERE source_key = "R1" AND target_entity_ref = "A" + UNION + SELECT descendants.root_id, target_entity_ref + FROM descendants + JOIN refresh_state_references ON source_entity_ref = descendants.entity_ref + ), + -- All the nodes that can be reached upwards from the descendants + ancestors(root_id, via_entity_ref, to_entity_ref) AS ( + SELECT CAST(NULL as INT), entity_ref, entity_ref + FROM descendants + UNION + SELECT + CASE WHEN source_key IS NOT NULL THEN id ELSE NULL END, + source_entity_ref, + ancestors.to_entity_ref + FROM ancestors + JOIN refresh_state_references ON target_entity_ref = ancestors.via_entity_ref + ) + -- Start out with all of the descendants + SELECT descendants.entity_ref + FROM descendants + -- Expand with all ancestors that point to those, but aren't the current root + LEFT OUTER JOIN ancestors + ON ancestors.to_entity_ref = descendants.entity_ref + AND ancestors.root_id IS NOT NULL + AND ancestors.root_id != descendants.root_id + -- Exclude all lines that had such a foreign ancestor + WHERE ancestors.root_id IS NULL; + */ + removedCount += await tx('refresh_state') + .whereIn('entity_ref', function orphanedEntityRefs(orphans) { + return ( + orphans + // All the nodes that can be reached downwards from our root + .withRecursive('descendants', function descendants(outer) { + return outer + .select({ root_id: 'id', entity_ref: 'target_entity_ref' }) + .from('refresh_state_references') + .where('source_key', sourceKey) + .whereIn('target_entity_ref', refs) + .union(function recursive(inner) { + return inner + .select({ + root_id: 'descendants.root_id', + entity_ref: 'refresh_state_references.target_entity_ref', + }) + .from('descendants') + .join('refresh_state_references', { + 'descendants.entity_ref': + 'refresh_state_references.source_entity_ref', + }); + }); + }) + // All the nodes that can be reached upwards from the descendants + .withRecursive('ancestors', function ancestors(outer) { + return outer + .select({ + root_id: rootId(), + via_entity_ref: 'entity_ref', + to_entity_ref: 'entity_ref', + }) + .from('descendants') + .union(function recursive(inner) { + return inner + .select({ + root_id: tx.raw( + 'CASE WHEN source_key IS NOT NULL THEN id ELSE NULL END', + [], + ), + via_entity_ref: 'source_entity_ref', + to_entity_ref: 'ancestors.to_entity_ref', + }) + .from('ancestors') + .join('refresh_state_references', { + target_entity_ref: 'ancestors.via_entity_ref', + }); + }); + }) + // Start out with all of the descendants + .select('descendants.entity_ref') + .from('descendants') + // Expand with all ancestors that point to those, but aren't the current root + .leftOuterJoin('ancestors', function keepaliveRoots() { + this.on('ancestors.to_entity_ref', '=', 'descendants.entity_ref'); + this.andOnNotNull('ancestors.root_id'); + this.andOn('ancestors.root_id', '!=', 'descendants.root_id'); + }) + .whereNull('ancestors.root_id') + ); + }) + .delete(); + + // Delete the references that originate only from this entity provider. Note + // that there may be more than one entity provider making a "claim" for a + // given root entity, if they emit with the same location key. + await tx('refresh_state_references') + .where('source_key', '=', sourceKey) + .whereIn('target_entity_ref', refs) + .delete(); + } + + return removedCount; +} diff --git a/plugins/catalog-backend/src/database/operations/provider/refreshByRefreshKeys.ts b/plugins/catalog-backend/src/database/operations/provider/refreshByRefreshKeys.ts new file mode 100644 index 0000000000..dc34c9cdf0 --- /dev/null +++ b/plugins/catalog-backend/src/database/operations/provider/refreshByRefreshKeys.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; +import { DbRefreshStateRow } from '../../tables'; + +/** + * Schedules a future refresh of entities, by so called "refresh keys" that may + * be associated with one or more entities. Note that this does not mean that + * the refresh happens immediately, but rather that their scheduling time gets + * moved up the queue and will get picked up eventually by the regular + * processing loop. + */ +export async function refreshByRefreshKeys(options: { + tx: Knex.Transaction; + keys: string[]; +}): Promise { + const { tx, keys } = options; + + await tx('refresh_state') + .whereIn('entity_id', function selectEntityRefs(inner) { + inner + .whereIn('key', keys) + .select({ + entity_id: 'refresh_keys.entity_id', + }) + .from('refresh_keys'); + }) + .update({ next_update_at: tx.fn.now() }); +} diff --git a/plugins/catalog-backend/src/database/operations/refreshState/checkLocationKeyConflict.ts b/plugins/catalog-backend/src/database/operations/refreshState/checkLocationKeyConflict.ts index a256b36729..97d376f9a4 100644 --- a/plugins/catalog-backend/src/database/operations/refreshState/checkLocationKeyConflict.ts +++ b/plugins/catalog-backend/src/database/operations/refreshState/checkLocationKeyConflict.ts @@ -23,11 +23,13 @@ import { DbRefreshStateRow } from '../../tables'; * * @returns The conflicting key if there is one. */ -export async function checkLocationKeyConflict( - tx: Knex.Transaction, - entityRef: string, - locationKey?: string, -): Promise { +export async function checkLocationKeyConflict(options: { + tx: Knex.Transaction; + entityRef: string; + locationKey?: string; +}): Promise { + const { tx, entityRef, locationKey } = options; + const row = await tx('refresh_state') .select('location_key') .where('entity_ref', entityRef) diff --git a/plugins/catalog-backend/src/database/operations/refreshState/insertUnprocessedEntity.ts b/plugins/catalog-backend/src/database/operations/refreshState/insertUnprocessedEntity.ts index bf34fd2ce8..8e1c6edc87 100644 --- a/plugins/catalog-backend/src/database/operations/refreshState/insertUnprocessedEntity.ts +++ b/plugins/catalog-backend/src/database/operations/refreshState/insertUnprocessedEntity.ts @@ -25,13 +25,15 @@ import { isDatabaseConflictError } from '@backstage/backend-common'; * Attempts to insert a new refresh state row for the given entity, returning * true if successful and false if there was a conflict. */ -export async function insertUnprocessedEntity( - tx: Knex.Transaction, - entity: Entity, - hash: string, - logger: Logger, - locationKey?: string, -): Promise { +export async function insertUnprocessedEntity(options: { + tx: Knex.Transaction; + entity: Entity; + hash: string; + locationKey?: string; + logger: Logger; +}): Promise { + const { tx, entity, hash, logger, locationKey } = options; + const entityRef = stringifyEntityRef(entity); const serializedEntity = JSON.stringify(entity); diff --git a/plugins/catalog-backend/src/database/operations/refreshState/updateUnprocessedEntity.ts b/plugins/catalog-backend/src/database/operations/refreshState/updateUnprocessedEntity.ts index 084454de66..e26db4429b 100644 --- a/plugins/catalog-backend/src/database/operations/refreshState/updateUnprocessedEntity.ts +++ b/plugins/catalog-backend/src/database/operations/refreshState/updateUnprocessedEntity.ts @@ -24,12 +24,14 @@ import { DbRefreshStateRow } from '../../tables'; * * Updating the entity will also cause it to be scheduled for immediate processing. */ -export async function updateUnprocessedEntity( - tx: Knex.Transaction, - entity: Entity, - hash: string, - locationKey?: string, -): Promise { +export async function updateUnprocessedEntity(options: { + tx: Knex.Transaction; + entity: Entity; + hash: string; + locationKey?: string; +}): Promise { + const { tx, entity, hash, locationKey } = options; + const entityRef = stringifyEntityRef(entity); const serializedEntity = JSON.stringify(entity); From 273ba3a77fd246a3a10ab150d8a35dadc5fdea62 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 9 Dec 2022 16:25:44 +0100 Subject: [PATCH 129/437] catalog: register OpenTelemetry metrics, deprecate prom-client Signed-off-by: Johan Haals --- .changeset/healthy-waves-compare.md | 5 + plugins/catalog-backend/package.json | 1 + .../DefaultCatalogProcessingEngine.ts | 99 +++++++++++++++---- yarn.lock | 70 ++++++++++++- 4 files changed, 152 insertions(+), 23 deletions(-) create mode 100644 .changeset/healthy-waves-compare.md diff --git a/.changeset/healthy-waves-compare.md b/.changeset/healthy-waves-compare.md new file mode 100644 index 0000000000..cdd19d1450 --- /dev/null +++ b/.changeset/healthy-waves-compare.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Deprecated Prometheus metrics in favour of OpenTelemtry metrics. diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index fe8ccc609d..e29c1d3c9a 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -47,6 +47,7 @@ "@backstage/plugin-scaffolder-common": "workspace:^", "@backstage/plugin-search-common": "workspace:^", "@backstage/types": "workspace:^", + "@opentelemetry/api": "^1.3.0", "@types/express": "^4.17.6", "codeowners-utils": "^1.0.2", "core-js": "^3.6.5", diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts index bef2878099..a27e5a0ce4 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts @@ -23,6 +23,7 @@ import { assertError, serializeError, stringifyError } from '@backstage/errors'; import { Hash } from 'crypto'; import stableStringify from 'fast-json-stable-stringify'; import { Logger } from 'winston'; +import { metrics } from '@opentelemetry/api'; import { ProcessingDatabase, RefreshStateItem } from '../database/types'; import { createCounterMetric, createSummaryMetric } from '../util/metrics'; import { @@ -257,62 +258,124 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine { // Helps wrap the timing and logging behaviors function progressTracker() { - const stitchedEntities = createCounterMetric({ + // prom-client metrics are deprecated in favour of OpenTelemetry metrics. + const promStitchedEntities = createCounterMetric({ name: 'catalog_stitched_entities_count', - help: 'Amount of entities stitched', + help: 'Amount of entities stitched. DEPRECATED, use OpenTelemetry metrics instead', }); - const processedEntities = createCounterMetric({ + const promProcessedEntities = createCounterMetric({ name: 'catalog_processed_entities_count', - help: 'Amount of entities processed', + help: 'Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead', labelNames: ['result'], }); - const processingDuration = createSummaryMetric({ + const promProcessingDuration = createSummaryMetric({ name: 'catalog_processing_duration_seconds', - help: 'Time spent executing the full processing flow', + help: 'Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead', labelNames: ['result'], }); - const processorsDuration = createSummaryMetric({ + const promProcessorsDuration = createSummaryMetric({ name: 'catalog_processors_duration_seconds', - help: 'Time spent executing catalog processors', + help: 'Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead', labelNames: ['result'], }); - const processingQueueDelay = createSummaryMetric({ + const promProcessingQueueDelay = createSummaryMetric({ name: 'catalog_processing_queue_delay_seconds', - help: 'The amount of delay between being scheduled for processing, and the start of actually being processed', + help: 'The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead', }); + const meter = metrics.getMeter('default'); + const stitchedEntities = meter.createCounter( + 'catalog.stitched.entities.count', + { + description: 'Amount of entities stitched', + }, + ); + + const processedEntities = meter.createCounter( + 'catalog.stitched.entities.count', + { description: 'Amount of entities processed' }, + ); + + const processingDuration = meter.createHistogram( + 'catalog.processing.duration', + { + description: 'Time spent executing the full processing flow', + unit: 'seconds', + }, + ); + + const processorsDuration = meter.createHistogram( + 'catalog.processors.duration', + { + description: 'Time spent executing catalog processors', + unit: 'seconds', + }, + ); + + const processingQueueDelay = meter.createHistogram( + 'catalog.processing.queue.delay', + { + description: + 'The amount of delay between being scheduled for processing, and the start of actually being processed', + unit: 'seconds', + }, + ); + function processStart(item: RefreshStateItem, logger: Logger) { + const startTime = process.hrtime(); + const endOverallTimer = promProcessingDuration.startTimer(); + const endProcessorsTimer = promProcessorsDuration.startTimer(); + logger.debug(`Processing ${item.entityRef}`); if (item.nextUpdateAt) { - processingQueueDelay.observe(-item.nextUpdateAt.diffNow().as('seconds')); + promProcessingQueueDelay.observe( + -item.nextUpdateAt.diffNow().as('seconds'), + ); + processingQueueDelay.record(-item.nextUpdateAt.diffNow().as('seconds')); } - const endOverallTimer = processingDuration.startTimer(); - const endProcessorsTimer = processorsDuration.startTimer(); + function endTime() { + const delta = process.hrtime(startTime); + return delta[0] + delta[1] / 1e9; + } function markProcessorsCompleted(result: EntityProcessingResult) { endProcessorsTimer({ result: result.ok ? 'ok' : 'failed' }); + processorsDuration.record(endTime(), { + result: result.ok ? 'ok' : 'failed', + }); } function markSuccessfulWithNoChanges() { endOverallTimer({ result: 'unchanged' }); - processedEntities.inc({ result: 'unchanged' }, 1); + promProcessedEntities.inc({ result: 'unchanged' }, 1); + + processingDuration.record(endTime(), { result: 'unchanged' }); + processedEntities.add(1, { result: 'unchanged' }); } function markSuccessfulWithErrors() { endOverallTimer({ result: 'errors' }); - processedEntities.inc({ result: 'errors' }, 1); + promProcessedEntities.inc({ result: 'errors' }, 1); + + processingDuration.record(endTime(), { result: 'errors' }); + processedEntities.add(1, { result: 'errors' }); } function markSuccessfulWithChanges(stitchedCount: number) { endOverallTimer({ result: 'changed' }); - stitchedEntities.inc(stitchedCount); - processedEntities.inc({ result: 'changed' }, 1); + promStitchedEntities.inc(stitchedCount); + promProcessedEntities.inc({ result: 'changed' }, 1); + + processingDuration.record(endTime(), { result: 'changed' }); + stitchedEntities.add(stitchedCount); + processedEntities.add(1, { result: 'changed' }); } function markFailed(error: Error) { - processedEntities.inc({ result: 'failed' }, 1); + promProcessedEntities.inc({ result: 'failed' }, 1); + processedEntities.add(1, { result: 'failed' }); logger.warn(`Processing of ${item.entityRef} failed`, error); } diff --git a/yarn.lock b/yarn.lock index 84aaed0d0c..5a4116ed74 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5215,6 +5215,7 @@ __metadata: "@backstage/plugin-search-backend-node": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@backstage/types": "workspace:^" + "@opentelemetry/api": ^1.3.0 "@types/core-js": ^2.5.4 "@types/express": ^4.17.6 "@types/git-url-parse": ^9.0.0 @@ -12218,10 +12219,66 @@ __metadata: languageName: node linkType: hard -"@opentelemetry/api@npm:^1.0.1": - version: 1.0.4 - resolution: "@opentelemetry/api@npm:1.0.4" - checksum: 793e9b5c21666b647a60c58c46c3e00ad1dac38505102b026ad0ef617571d637aca54a18533a73c1e288c95b5ac77e2db17f96467f11833ac1165338e1184260 +"@opentelemetry/api@npm:^1.0.1, @opentelemetry/api@npm:^1.3.0": + version: 1.3.0 + resolution: "@opentelemetry/api@npm:1.3.0" + checksum: 33d284b67b6fab20ff72961d289c6487d3cb27caf7489f0231d7030551f82871e081e744b0390751d8aef3bf1614bd79f854788901a354e15274f552581fb374 + languageName: node + linkType: hard + +"@opentelemetry/core@npm:1.8.0": + version: 1.8.0 + resolution: "@opentelemetry/core@npm:1.8.0" + dependencies: + "@opentelemetry/semantic-conventions": 1.8.0 + peerDependencies: + "@opentelemetry/api": ">=1.0.0 <1.4.0" + checksum: 09cd58ec764f97b175af47fbaff335f06a31914fcb59b30c4f96f6ba69391ce4e59e3da46a95fd15a7d0e8fc5c638195aba95ca1a916127207481f1283991c97 + languageName: node + linkType: hard + +"@opentelemetry/exporter-prometheus@npm:^0.34.0": + version: 0.34.0 + resolution: "@opentelemetry/exporter-prometheus@npm:0.34.0" + dependencies: + "@opentelemetry/core": 1.8.0 + "@opentelemetry/resources": 1.8.0 + "@opentelemetry/sdk-metrics": 1.8.0 + peerDependencies: + "@opentelemetry/api": ^1.3.0 + checksum: 6c17d5ec1c638fb5f561230f706d84866d504aa2805ed74ab2174dae9a1c9d39ce909b61ba9796d38fbcdef85f76805e23d5c80d762231e08384e032beb38b65 + languageName: node + linkType: hard + +"@opentelemetry/resources@npm:1.8.0": + version: 1.8.0 + resolution: "@opentelemetry/resources@npm:1.8.0" + dependencies: + "@opentelemetry/core": 1.8.0 + "@opentelemetry/semantic-conventions": 1.8.0 + peerDependencies: + "@opentelemetry/api": ">=1.0.0 <1.4.0" + checksum: eeea7864c486d31679dbae3c31e9badf277ece24ba8bd063e2bcc34b2d7240f543678b8106743aff30c6b0f028f2bed286d64cfd7b5cde6f9a0f6a9271a3fce1 + languageName: node + linkType: hard + +"@opentelemetry/sdk-metrics@npm:1.8.0, @opentelemetry/sdk-metrics@npm:^1.8.0": + version: 1.8.0 + resolution: "@opentelemetry/sdk-metrics@npm:1.8.0" + dependencies: + "@opentelemetry/core": 1.8.0 + "@opentelemetry/resources": 1.8.0 + lodash.merge: 4.6.2 + peerDependencies: + "@opentelemetry/api": ">=1.3.0 <1.4.0" + checksum: 8dfb82e70b14fe2e95ce3f3d0b18e42981bfabe64c63b7f8c429aed197815356836a0350ad5cf38696f561c616ff5572f2ca63f4f968ae1367ed7180f730cad7 + languageName: node + linkType: hard + +"@opentelemetry/semantic-conventions@npm:1.8.0": + version: 1.8.0 + resolution: "@opentelemetry/semantic-conventions@npm:1.8.0" + checksum: df30ad9486b6c611c4110fab80815301a7cc9cb320983d6c5792a1b411dc4e4f04c489b2abfdea0da7f7bbb27b9f3c456764ba07f23432900a9bbcbc5f98ff58 languageName: node linkType: hard @@ -22007,6 +22064,9 @@ __metadata: "@backstage/plugin-todo-backend": "workspace:^" "@gitbeaker/node": ^35.1.0 "@octokit/rest": ^19.0.3 + "@opentelemetry/api": ^1.3.0 + "@opentelemetry/exporter-prometheus": ^0.34.0 + "@opentelemetry/sdk-metrics": ^1.8.0 "@types/dockerode": ^3.3.0 "@types/express": ^4.17.6 "@types/express-serve-static-core": ^4.17.5 @@ -27627,7 +27687,7 @@ __metadata: languageName: node linkType: hard -"lodash.merge@npm:^4.6.2": +"lodash.merge@npm:4.6.2, lodash.merge@npm:^4.6.2": version: 4.6.2 resolution: "lodash.merge@npm:4.6.2" checksum: ad580b4bdbb7ca1f7abf7e1bce63a9a0b98e370cf40194b03380a46b4ed799c9573029599caebc1b14e3f24b111aef72b96674a56cfa105e0f5ac70546cdc005 From 0053d07bee06e1addd1120d400d384156a0c6d90 Mon Sep 17 00:00:00 2001 From: Miguel Alexandre Date: Fri, 9 Dec 2022 16:26:17 +0100 Subject: [PATCH 130/437] Allow setting dismissStaleReviews on github:publish action Signed-off-by: Miguel Alexandre --- .changeset/witty-wasps-kiss.md | 5 ++ .../builtin/github/githubRepoPush.test.ts | 66 ++++++++++++++++++ .../actions/builtin/github/githubRepoPush.ts | 4 ++ .../actions/builtin/github/helpers.ts | 2 + .../actions/builtin/github/inputProperties.ts | 7 ++ .../src/scaffolder/actions/builtin/helpers.ts | 3 + .../actions/builtin/publish/github.test.ts | 69 +++++++++++++++++++ .../actions/builtin/publish/github.ts | 4 ++ 8 files changed, 160 insertions(+) create mode 100644 .changeset/witty-wasps-kiss.md diff --git a/.changeset/witty-wasps-kiss.md b/.changeset/witty-wasps-kiss.md new file mode 100644 index 0000000000..ef51ef7bfd --- /dev/null +++ b/.changeset/witty-wasps-kiss.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': minor +--- + +Update the `github:publish` action to allow passing wether to dismiss stale reviews on the protected default branch. diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts index fc39e03cbc..dfd3297b87 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts @@ -495,4 +495,70 @@ describe('github:repo:push', () => { expect(enableBranchProtectionOnDefaultRepoBranch).not.toHaveBeenCalled(); }); + + it('should call enableBranchProtectionOnDefaultRepoBranch with the correct values of dismissStaleReviews', async () => { + mockOctokit.rest.repos.get.mockResolvedValue({ + data: { + clone_url: 'https://github.com/clone/url.git', + html_url: 'https://github.com/html/url', + }, + }); + + await action.handler(mockContext); + + expect(enableBranchProtectionOnDefaultRepoBranch).toHaveBeenCalledWith({ + owner: 'owner', + client: mockOctokit, + repoName: 'repository', + logger: mockContext.logger, + defaultBranch: 'master', + requireCodeOwnerReviews: false, + requiredStatusCheckContexts: [], + requireBranchesToBeUpToDate: true, + enforceAdmins: true, + dismissStaleReviews: false, + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + dismissStaleReviews: true, + }, + }); + + expect(enableBranchProtectionOnDefaultRepoBranch).toHaveBeenCalledWith({ + owner: 'owner', + client: mockOctokit, + repoName: 'repository', + logger: mockContext.logger, + defaultBranch: 'master', + requireCodeOwnerReviews: false, + requiredStatusCheckContexts: [], + requireBranchesToBeUpToDate: true, + enforceAdmins: true, + dismissStaleReviews: true, + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + dismissStaleReviews: false, + }, + }); + + expect(enableBranchProtectionOnDefaultRepoBranch).toHaveBeenCalledWith({ + owner: 'owner', + client: mockOctokit, + repoName: 'repository', + logger: mockContext.logger, + defaultBranch: 'master', + requireCodeOwnerReviews: false, + requiredStatusCheckContexts: [], + requireBranchesToBeUpToDate: true, + enforceAdmins: true, + dismissStaleReviews: false, + }); + }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.ts index 1694e37710..4c931e029c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.ts @@ -49,6 +49,7 @@ export function createGithubRepoPushAction(options: { gitAuthorName?: string; gitAuthorEmail?: string; requireCodeOwnerReviews?: boolean; + dismissStaleReviews?: boolean; bypassPullRequestAllowances?: | { users?: string[]; @@ -71,6 +72,7 @@ export function createGithubRepoPushAction(options: { properties: { repoUrl: inputProps.repoUrl, requireCodeOwnerReviews: inputProps.requireCodeOwnerReviews, + dismissStaleReviews: inputProps.dismissStaleReviews, requiredStatusCheckContexts: inputProps.requiredStatusCheckContexts, bypassPullRequestAllowances: inputProps.bypassPullRequestAllowances, requireBranchesToBeUpToDate: inputProps.requireBranchesToBeUpToDate, @@ -102,6 +104,7 @@ export function createGithubRepoPushAction(options: { gitAuthorName, gitAuthorEmail, requireCodeOwnerReviews = false, + dismissStaleReviews = false, bypassPullRequestAllowances, requiredStatusCheckContexts = [], requireBranchesToBeUpToDate = true, @@ -148,6 +151,7 @@ export function createGithubRepoPushAction(options: { gitCommitMessage, gitAuthorName, gitAuthorEmail, + dismissStaleReviews, ); ctx.output('remoteUrl', remoteUrl); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/helpers.ts index fe29378e2e..4dc7034f18 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/helpers.ts @@ -263,6 +263,7 @@ export async function initRepoPushAndProtect( gitCommitMessage?: string, gitAuthorName?: string, gitAuthorEmail?: string, + dismissStaleReviews?: boolean, ) { const gitAuthorInfo = { name: gitAuthorName @@ -303,6 +304,7 @@ export async function initRepoPushAndProtect( requiredStatusCheckContexts, requireBranchesToBeUpToDate, enforceAdmins: protectEnforceAdmins, + dismissStaleReviews: dismissStaleReviews, }); } catch (e) { assertError(e); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/inputProperties.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/inputProperties.ts index e832c3dbfa..6c3bebf47e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/inputProperties.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/inputProperties.ts @@ -38,6 +38,12 @@ const requireCodeOwnerReviews = { 'Require an approved review in PR including files with a designated Code Owner', type: 'boolean', }; +const dismissStaleReviews = { + title: 'Dismiss Stale Reviews', + description: + 'New reviewable commits pushed to a matching branch will dismiss pull request review approvals.', + type: 'boolean', +}; const requiredStatusCheckContexts = { title: 'Required Status Check Contexts', description: @@ -207,6 +213,7 @@ export { bypassPullRequestAllowances }; export { repoUrl }; export { repoVisibility }; export { requireCodeOwnerReviews }; +export { dismissStaleReviews }; export { requiredStatusCheckContexts }; export { requireBranchesToBeUpToDate }; export { sourcePath }; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts index 5afe51cc35..a7d29ec756 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts @@ -193,6 +193,7 @@ type BranchProtectionOptions = { requireBranchesToBeUpToDate?: boolean; defaultBranch?: string; enforceAdmins?: boolean; + dismissStaleReviews?: boolean; }; export const enableBranchProtectionOnDefaultRepoBranch = async ({ @@ -206,6 +207,7 @@ export const enableBranchProtectionOnDefaultRepoBranch = async ({ requireBranchesToBeUpToDate = true, defaultBranch = 'master', enforceAdmins = true, + dismissStaleReviews = false, }: BranchProtectionOptions): Promise => { const tryOnce = async () => { try { @@ -233,6 +235,7 @@ export const enableBranchProtectionOnDefaultRepoBranch = async ({ required_approving_review_count: 1, require_code_owner_reviews: requireCodeOwnerReviews, bypass_pull_request_allowances: bypassPullRequestAllowances, + dismiss_stale_reviews: dismissStaleReviews, }, }); } catch (e) { diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts index cebc968669..6c4cd55d98 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts @@ -922,4 +922,73 @@ describe('publish:github', () => { names: ['node.js'], }); }); + + it('should call enableBranchProtectionOnDefaultRepoBranch with the correct values of dismissStaleReviews', async () => { + mockOctokit.rest.users.getByUsername.mockResolvedValue({ + data: { type: 'User' }, + }); + + mockOctokit.rest.repos.createForAuthenticatedUser.mockResolvedValue({ + data: { + name: 'repo', + }, + }); + + await action.handler(mockContext); + + expect(enableBranchProtectionOnDefaultRepoBranch).toHaveBeenCalledWith({ + owner: 'owner', + client: mockOctokit, + repoName: 'repo', + logger: mockContext.logger, + defaultBranch: 'master', + requireCodeOwnerReviews: false, + requiredStatusCheckContexts: [], + requireBranchesToBeUpToDate: true, + enforceAdmins: true, + dismissStaleReviews: false, + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + dismissStaleReviews: true, + }, + }); + + expect(enableBranchProtectionOnDefaultRepoBranch).toHaveBeenCalledWith({ + owner: 'owner', + client: mockOctokit, + repoName: 'repo', + logger: mockContext.logger, + defaultBranch: 'master', + requireCodeOwnerReviews: false, + requiredStatusCheckContexts: [], + requireBranchesToBeUpToDate: true, + enforceAdmins: true, + dismissStaleReviews: true, + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + dismissStaleReviews: false, + }, + }); + + expect(enableBranchProtectionOnDefaultRepoBranch).toHaveBeenCalledWith({ + owner: 'owner', + client: mockOctokit, + repoName: 'repo', + logger: mockContext.logger, + defaultBranch: 'master', + requireCodeOwnerReviews: false, + requiredStatusCheckContexts: [], + requireBranchesToBeUpToDate: true, + enforceAdmins: true, + dismissStaleReviews: false, + }); + }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index be4cbb1dad..1d3cce8067 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -68,6 +68,7 @@ export function createPublishGithubAction(options: { } | undefined; requireCodeOwnerReviews?: boolean; + dismissStaleReviews?: boolean; requiredStatusCheckContexts?: string[]; requireBranchesToBeUpToDate?: boolean; repoVisibility?: 'private' | 'internal' | 'public'; @@ -103,6 +104,7 @@ export function createPublishGithubAction(options: { access: inputProps.access, bypassPullRequestAllowances: inputProps.bypassPullRequestAllowances, requireCodeOwnerReviews: inputProps.requireCodeOwnerReviews, + dismissStaleReviews: inputProps.dismissStaleReviews, requiredStatusCheckContexts: inputProps.requiredStatusCheckContexts, requireBranchesToBeUpToDate: inputProps.requireBranchesToBeUpToDate, repoVisibility: inputProps.repoVisibility, @@ -138,6 +140,7 @@ export function createPublishGithubAction(options: { homepage, access, requireCodeOwnerReviews = false, + dismissStaleReviews = false, bypassPullRequestAllowances, requiredStatusCheckContexts = [], requireBranchesToBeUpToDate = true, @@ -213,6 +216,7 @@ export function createPublishGithubAction(options: { gitCommitMessage, gitAuthorName, gitAuthorEmail, + dismissStaleReviews, ); ctx.output('remoteUrl', remoteUrl); From 0f3ae75ee767508893df5ff5f74c4a37080a0436 Mon Sep 17 00:00:00 2001 From: Miguel Alexandre Date: Fri, 9 Dec 2022 16:40:32 +0100 Subject: [PATCH 131/437] Update api-reports for plugins/scaffolder-backend Signed-off-by: Miguel Alexandre --- plugins/scaffolder-backend/api-report.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 1b5ea877b8..fd5af19fe2 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -243,6 +243,7 @@ export function createGithubRepoPushAction(options: { gitAuthorName?: string | undefined; gitAuthorEmail?: string | undefined; requireCodeOwnerReviews?: boolean | undefined; + dismissStaleReviews?: boolean | undefined; bypassPullRequestAllowances?: | { users?: string[]; @@ -388,6 +389,7 @@ export function createPublishGithubAction(options: { } | undefined; requireCodeOwnerReviews?: boolean | undefined; + dismissStaleReviews?: boolean | undefined; requiredStatusCheckContexts?: string[] | undefined; requireBranchesToBeUpToDate?: boolean | undefined; repoVisibility?: 'internal' | 'private' | 'public' | undefined; From dcce6f5d10d3381f9883d9c60c4afc9705046057 Mon Sep 17 00:00:00 2001 From: Miguel Alexandre Date: Fri, 9 Dec 2022 16:48:04 +0100 Subject: [PATCH 132/437] Fix tests Signed-off-by: Miguel Alexandre --- .../actions/builtin/github/githubRepoPush.test.ts | 10 ++++++++++ .../scaffolder/actions/builtin/publish/github.test.ts | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts index dfd3297b87..46f33a823d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/github/githubRepoPush.test.ts @@ -285,6 +285,7 @@ describe('github:repo:push', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); await action.handler({ @@ -305,6 +306,7 @@ describe('github:repo:push', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); await action.handler({ @@ -325,6 +327,7 @@ describe('github:repo:push', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); }); @@ -348,6 +351,7 @@ describe('github:repo:push', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); await action.handler({ @@ -368,6 +372,7 @@ describe('github:repo:push', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); await action.handler({ @@ -388,6 +393,7 @@ describe('github:repo:push', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: false, + dismissStaleReviews: false, }); }); @@ -411,6 +417,7 @@ describe('github:repo:push', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); await action.handler({ @@ -432,6 +439,7 @@ describe('github:repo:push', () => { requiredStatusCheckContexts: ['statusCheck'], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); await action.handler({ @@ -453,6 +461,7 @@ describe('github:repo:push', () => { requiredStatusCheckContexts: ['statusCheck'], requireBranchesToBeUpToDate: false, enforceAdmins: true, + dismissStaleReviews: false, }); await action.handler({ @@ -474,6 +483,7 @@ describe('github:repo:push', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts index 6c4cd55d98..b5bda4c5d0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts @@ -671,6 +671,7 @@ describe('publish:github', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); await action.handler({ @@ -691,6 +692,7 @@ describe('publish:github', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); await action.handler({ @@ -711,6 +713,7 @@ describe('publish:github', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); }); @@ -737,6 +740,7 @@ describe('publish:github', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); await action.handler({ @@ -757,6 +761,7 @@ describe('publish:github', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: false, + dismissStaleReviews: false, }); await action.handler({ @@ -777,6 +782,7 @@ describe('publish:github', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); }); @@ -803,6 +809,7 @@ describe('publish:github', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); await action.handler({ @@ -824,6 +831,7 @@ describe('publish:github', () => { requiredStatusCheckContexts: ['statusCheck'], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); await action.handler({ @@ -845,6 +853,7 @@ describe('publish:github', () => { requiredStatusCheckContexts: ['statusCheck'], requireBranchesToBeUpToDate: false, enforceAdmins: true, + dismissStaleReviews: false, }); await action.handler({ @@ -865,6 +874,7 @@ describe('publish:github', () => { requiredStatusCheckContexts: [], requireBranchesToBeUpToDate: true, enforceAdmins: true, + dismissStaleReviews: false, }); }); From 88d64d3e4cd2cd6bc9fb326dbf4035d5c9a9713d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 9 Dec 2022 17:07:39 +0100 Subject: [PATCH 133/437] Update plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: Patrik Oldsberg --- .../operations/provider/deleteWithEagerPruningOfChildren.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.ts b/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.ts index 5e2aa482a3..3d642cbf3a 100644 --- a/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.ts +++ b/plugins/catalog-backend/src/database/operations/provider/deleteWithEagerPruningOfChildren.ts @@ -37,7 +37,7 @@ export async function deleteWithEagerPruningOfChildren(options: { tx.raw( tx.client.config.client.includes('mysql') ? 'CAST(NULL as UNSIGNED INT)' - : 'CAST(NULL as UNSIGNED INT)', + : 'CAST(NULL as INT)', [], ); From 71147d5c16703c94611f9c9a2b7f1d4ef9b8b012 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 9 Dec 2022 17:27:02 +0100 Subject: [PATCH 134/437] changesets: added changeset for catalog reorg Signed-off-by: Patrik Oldsberg --- .changeset/lucky-chicken-greet.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/lucky-chicken-greet.md diff --git a/.changeset/lucky-chicken-greet.md b/.changeset/lucky-chicken-greet.md new file mode 100644 index 0000000000..7e061dc040 --- /dev/null +++ b/.changeset/lucky-chicken-greet.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Internal code reorganization. From 24ff18621cdf08f1a0d806f0f08198c6e0fe3cb1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 9 Dec 2022 17:27:56 +0100 Subject: [PATCH 135/437] catalog-backend: reference cleanup fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: Patrik Oldsberg --- .../database/DefaultProcessingDatabase.test.ts | 17 +++++++++++------ .../src/database/DefaultProcessingDatabase.ts | 3 --- .../src/database/DefaultProviderDatabase.ts | 5 +++++ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 4fbb38b4fc..540c93ef13 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -462,12 +462,17 @@ describe('DefaultProcessingDatabase', () => { knexTx( 'refresh_state_references', ).select(), - ).resolves.toEqual([ - expect.objectContaining({ - source_entity_ref: 'location:default/fakelocation', - target_entity_ref: 'component:default/1', - }), - ]); + ).resolves.toEqual( + step.expectConflict + ? [] + : [ + // eslint-disable-next-line jest/no-conditional-expect + expect.objectContaining({ + source_entity_ref: 'location:default/fakelocation', + target_entity_ref: 'component:default/1', + }), + ], + ); expect(mockLogger.error).not.toHaveBeenCalled(); } diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 1127deed7e..761f5bfd72 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -314,7 +314,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards const stateReferences = new Array(); - const conflictingStateReferences = new Array(); // Upsert all of the unprocessed entities into the refresh_state table, by // their entity ref. @@ -357,13 +356,11 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { this.options.logger.warn( `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`, ); - conflictingStateReferences.push(entityRef); } } // Replace all references for the originating entity or source and then create new ones await tx('refresh_state_references') - .whereNotIn('target_entity_ref', conflictingStateReferences) .andWhere({ source_entity_ref: options.sourceEntityRef }) .delete(); await tx.batchInsert( diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts index f1073fd984..a0d376327b 100644 --- a/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.ts @@ -161,6 +161,11 @@ export class DefaultProviderDatabase implements ProviderDatabase { }); } + await tx('refresh_state_references') + .where('target_entity_ref', entityRef) + .andWhere({ source_key: options.sourceKey }) + .delete(); + if (ok) { await tx( 'refresh_state_references', From 22e51086eb761e9b4354d452ba6084d1c8b49a17 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 9 Dec 2022 17:31:18 +0100 Subject: [PATCH 136/437] catalog-backend: add failing test from #15111 Signed-off-by: Patrik Oldsberg --- .../database/DefaultProviderDatabase.test.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts index 43f2209b60..140925b6fd 100644 --- a/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProviderDatabase.test.ts @@ -610,6 +610,23 @@ describe('DefaultProviderDatabase', () => { }), ]), ); + let references = await knex( + 'refresh_state_references', + ).select(); + expect(references).toEqual([ + { + id: 1, + source_key: 'lols', + source_entity_ref: null, + target_entity_ref: 'component:default/a', + }, + { + id: 2, + source_key: 'lols', + source_entity_ref: null, + target_entity_ref: 'component:default/b', + }, + ]); await db.transaction(async tx => { await db.replaceUnprocessedEntities(tx, { @@ -653,6 +670,23 @@ describe('DefaultProviderDatabase', () => { }), ]), ); + references = await knex( + 'refresh_state_references', + ).select(); + expect(references).toEqual([ + { + id: 2, + source_key: 'lols', + source_entity_ref: null, + target_entity_ref: 'component:default/b', + }, + { + id: 3, + source_key: 'lols', + source_entity_ref: null, + target_entity_ref: 'component:default/a', + }, + ]); }, 60_000, ); From 818176b841ae12f9a8ab801a0f8b7b850957b8ea Mon Sep 17 00:00:00 2001 From: Claire Casey Date: Fri, 9 Dec 2022 11:28:31 -0500 Subject: [PATCH 137/437] add CTA for office hours Signed-off-by: Claire Casey --- microsite/pages/en/index.js | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/microsite/pages/en/index.js b/microsite/pages/en/index.js index ab3dabe14e..b7eb3a9cf0 100644 --- a/microsite/pages/en/index.js +++ b/microsite/pages/en/index.js @@ -45,13 +45,21 @@ class Index extends React.Component { ship high-quality code quickly — without compromising autonomy. - - GitHub - + + + GitHub + + + Office Hours + + Date: Fri, 9 Dec 2022 17:32:25 +0100 Subject: [PATCH 138/437] changesets: added changeset for catalog reference fix Signed-off-by: Patrik Oldsberg --- .changeset/tasty-impalas-mix.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/tasty-impalas-mix.md diff --git a/.changeset/tasty-impalas-mix.md b/.changeset/tasty-impalas-mix.md new file mode 100644 index 0000000000..c830c45c56 --- /dev/null +++ b/.changeset/tasty-impalas-mix.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Fixed an issue where internal references in the catalog would stick around for longer than expected, causing entities to not be deleted or orphaned as expected. From dfb269fab2571d3f96d486e0493efbd0012a636a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 9 Dec 2022 18:08:08 +0100 Subject: [PATCH 139/437] create-app: comment out the default test proxy endpoint Signed-off-by: Patrik Oldsberg --- .changeset/odd-moles-notice.md | 5 +++++ .../create-app/templates/default-app/app-config.yaml.hbs | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 .changeset/odd-moles-notice.md diff --git a/.changeset/odd-moles-notice.md b/.changeset/odd-moles-notice.md new file mode 100644 index 0000000000..8cc82c93dc --- /dev/null +++ b/.changeset/odd-moles-notice.md @@ -0,0 +1,5 @@ +--- +'@backstage/create-app': patch +--- + +Updated the template to have the `'/test'` proxy endpoint in `app-config.yaml` be commented out by default. diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index 1a45d4015b..3b5de8baa3 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -46,9 +46,11 @@ integrations: # token: ${GHE_TOKEN} proxy: - '/test': - target: 'https://example.com' - changeOrigin: true + ### Example for how to add a proxy endpoint for the frontend. + ### A typical reason to do this is to handle HTTPS and CORS for internal services. + # '/test': + # target: 'https://example.com' + # changeOrigin: true # Reference documentation http://backstage.io/docs/features/techdocs/configuration # Note: After experimenting with basic setup, use CI/CD to generate docs From 5a17cc9b41a5e591b9e4e6834fb66184a54d9d7d Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Fri, 9 Dec 2022 11:22:12 -0600 Subject: [PATCH 140/437] adding documentation for exact locations Signed-off-by: Justin De Burgo --- plugins/catalog-backend/config.d.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 5f0f8f1ea2..cff8ee71c3 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -57,11 +57,15 @@ export interface Config { /** * The exact location, e.g. * "https://github.com/org/repo/blob/master/users.yaml". + * + * The exact location can also be used to match on locations + * that contain glob characters themselves, e.g. + * "https://github.com/org/*\/blob/master/*.yaml". */ exact?: string; /** * The pattern allowed for the location, e.g. - * "https://github.com/org/*/blob/master/*.yaml. + * "https://github.com/org/*\/blob/master/*.yaml". */ pattern?: string; }>; From eb050c324f4231d6eee3701d8a6c274c2eb9d7a1 Mon Sep 17 00:00:00 2001 From: Sergio Date: Wed, 7 Dec 2022 10:08:57 -0500 Subject: [PATCH 141/437] Add Service Maturity plugin to marketplace Signed-off-by: Sergio --- microsite/data/plugins/opslevel-maturity.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 microsite/data/plugins/opslevel-maturity.yaml diff --git a/microsite/data/plugins/opslevel-maturity.yaml b/microsite/data/plugins/opslevel-maturity.yaml new file mode 100644 index 0000000000..724f46ae17 --- /dev/null +++ b/microsite/data/plugins/opslevel-maturity.yaml @@ -0,0 +1,14 @@ +--- +title: Service Maturity +author: OpsLevel +authorUrl: https://www.opslevel.com/ +category: Quality +description: Integrate with OpsLevel to track performance against your engineering best practices and service maturity. +documentation: https://github.com/OpsLevel/backstage-plugin +iconUrl: https://avatars.githubusercontent.com/u/44910550?s=200&v=4 +npmPackageName: backstage-plugin-opslevel-maturity +tags: + - service maturity + - service quality + - maturity score +addedDate: '2022-12-07' From 5be74dcb5945fc33f373c355e088209562d57843 Mon Sep 17 00:00:00 2001 From: Sarah Medeiros Date: Fri, 9 Dec 2022 14:26:33 -0500 Subject: [PATCH 142/437] fix failing tests Signed-off-by: Sarah Medeiros --- .../EntityLifecyclePicker/EntityLifecyclePicker.test.tsx | 3 +++ .../components/EntityOwnerPicker/EntityOwnerPicker.test.tsx | 2 ++ 2 files changed, 5 insertions(+) diff --git a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.test.tsx b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.test.tsx index f925625b23..d1fab823be 100644 --- a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.test.tsx +++ b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.test.tsx @@ -169,6 +169,7 @@ describe('', () => { value={{ updateFilters, queryParameters: { lifecycles: ['experimental'] }, + backendEntities: sampleEntities, }} > @@ -182,6 +183,8 @@ describe('', () => { value={{ updateFilters, queryParameters: { lifecycles: ['production'] }, + backendEntities: sampleEntities, + q, }} > diff --git a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.test.tsx b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.test.tsx index c816810b10..6eebaa2409 100644 --- a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.test.tsx +++ b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.test.tsx @@ -183,6 +183,7 @@ describe('', () => { value={{ updateFilters, queryParameters: { owners: ['team-a'] }, + backendEntities: sampleEntities, }} > @@ -196,6 +197,7 @@ describe('', () => { value={{ updateFilters, queryParameters: { owners: ['team-b'] }, + backendEntities: sampleEntities, }} > From 7a7073f75c70f43a4a9334bfb98c130caffe495e Mon Sep 17 00:00:00 2001 From: Sarah Medeiros Date: Fri, 9 Dec 2022 14:57:00 -0500 Subject: [PATCH 143/437] Add test for new functionality Signed-off-by: Sarah Medeiros --- .../EntityLifecyclePicker.test.tsx | 18 +++++++++++++++++- .../EntityOwnerPicker.test.tsx | 17 +++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.test.tsx b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.test.tsx index d1fab823be..ef77f7615f 100644 --- a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.test.tsx +++ b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.test.tsx @@ -184,7 +184,6 @@ describe('', () => { updateFilters, queryParameters: { lifecycles: ['production'] }, backendEntities: sampleEntities, - q, }} > @@ -194,4 +193,21 @@ describe('', () => { lifecycles: new EntityLifecycleFilter(['production']), }); }); + it('removes lifecycles from filters if there are no available lifecycles', () => { + const updateFilters = jest.fn(); + render( + + + , + ); + expect(updateFilters).toHaveBeenLastCalledWith({ + lifecycles: undefined, + }); + }); }); diff --git a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.test.tsx b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.test.tsx index 6eebaa2409..b6e928b519 100644 --- a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.test.tsx +++ b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.test.tsx @@ -207,4 +207,21 @@ describe('', () => { owners: new EntityOwnerFilter(['team-b']), }); }); + it('removes owners from filters if there are none available', () => { + const updateFilters = jest.fn(); + render( + + + , + ); + expect(updateFilters).toHaveBeenLastCalledWith({ + owners: undefined, + }); + }); }); From 2e701b3796cc8b0b2a476e0c8abcafedeb0eea7e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 8 Dec 2022 21:18:01 +0100 Subject: [PATCH 144/437] refactor react-router -> react-router-dom Signed-off-by: Patrik Oldsberg --- .changeset/long-eyes-confess.md | 38 +++++++++++++++++++ packages/app-defaults/package.json | 1 - packages/app/package.json | 1 - packages/app/src/App.tsx | 2 +- packages/core-components/package.json | 1 - .../src/components/Button/Button.test.tsx | 2 +- .../src/components/Link/Link.test.tsx | 2 +- .../ProgressBars/GaugeCard.stories.tsx | 2 +- .../TabbedLayout/RoutedTabs.test.tsx | 2 +- .../components/TabbedLayout/RoutedTabs.tsx | 7 +++- .../TabbedLayout/TabbedLayout.stories.tsx | 2 +- .../src/layout/ErrorPage/ErrorPage.tsx | 2 +- .../src/layout/InfoCard/InfoCard.stories.tsx | 2 +- .../src/layout/ItemCard/ItemCard.stories.tsx | 2 +- .../src/layout/Sidebar/MobileSidebar.tsx | 2 +- .../layout/TabbedCard/TabbedCard.stories.tsx | 2 +- packages/dev-utils/package.json | 1 - packages/dev-utils/src/devApp/render.tsx | 2 +- .../techdocs-cli-embedded-app/package.json | 1 - .../techdocs-cli-embedded-app/src/App.tsx | 2 +- packages/test-utils/package.json | 1 - .../src/testUtils/appWrappers.test.tsx | 2 +- .../test-utils/src/testUtils/appWrappers.tsx | 2 +- plugins/airbrake/package.json | 2 +- plugins/airbrake/src/extensions.test.tsx | 2 +- plugins/api-docs/package.json | 1 - .../ApiExplorerPage/ApiExplorerPage.test.tsx | 2 +- .../ApiExplorerPage/ApiExplorerPage.tsx | 2 +- plugins/azure-devops/package.json | 2 +- .../BuildTable/BuildTable.stories.tsx | 2 +- .../PullRequestCard.stories.tsx | 2 +- plugins/badges/package.json | 2 +- plugins/badges/src/api/BadgesClient.ts | 2 +- plugins/catalog-graph/package.json | 2 +- .../CatalogGraphCard/CatalogGraphCard.tsx | 2 +- .../CatalogGraphPage/CatalogGraphPage.tsx | 2 +- .../useCatalogGraphPage.test.ts | 2 +- .../CatalogGraphPage/useCatalogGraphPage.ts | 2 +- plugins/catalog-import/package.json | 2 +- .../components/ImportPage/ImportPage.test.tsx | 2 +- .../src/components/ImportPage/ImportPage.tsx | 2 +- plugins/catalog-react/package.json | 2 +- .../components/AncestryPage.tsx | 2 +- .../src/hooks/useEntityListProvider.test.tsx | 2 +- .../src/hooks/useEntityListProvider.tsx | 2 +- plugins/catalog/package.json | 2 +- .../CatalogEntityPage/CatalogEntityPage.tsx | 2 +- .../CatalogEntityPage/useEntityFromUrl.ts | 2 +- .../CatalogPage/CatalogPage.test.tsx | 2 +- .../components/CatalogPage/CatalogPage.tsx | 2 +- .../components/EntityLayout/EntityLayout.tsx | 2 +- .../EntityOrphanWarning.tsx | 2 +- plugins/circleci/package.json | 1 - plugins/circleci/src/components/Router.tsx | 2 +- plugins/cloudbuild/package.json | 1 - plugins/cloudbuild/src/components/Router.tsx | 2 +- plugins/code-climate/package.json | 2 +- plugins/code-coverage/package.json | 1 - plugins/explore/package.json | 1 - .../ExplorePage/ExplorePage.test.tsx | 2 +- .../components/ExplorePage/ExplorePage.tsx | 2 +- plugins/git-release-manager/package.json | 2 +- .../src/features/RepoDetailsForm/Owner.tsx | 2 +- .../src/features/RepoDetailsForm/Repo.tsx | 2 +- .../RepoDetailsForm/VersioningStrategy.tsx | 2 +- .../src/hooks/useQueryHandler.ts | 2 +- plugins/github-actions/package.json | 1 - .../github-actions/src/components/Router.tsx | 2 +- plugins/home/package.json | 2 +- .../components/HomepageCompositionRoot.tsx | 2 +- plugins/jenkins/package.json | 1 - plugins/jenkins/src/components/Router.tsx | 2 +- plugins/kafka/package.json | 2 +- plugins/kafka/src/Router.tsx | 2 +- plugins/org/package.json | 1 - .../MembersList/MembersListCard.stories.tsx | 2 +- plugins/permission-react/package.json | 2 +- .../src/components/PermissionedRoute.tsx | 2 +- plugins/playlist/package.json | 1 - .../playlist/src/components/Router/Router.tsx | 2 +- .../src/hooks/usePlaylistList.test.tsx | 2 +- .../playlist/src/hooks/usePlaylistList.tsx | 2 +- plugins/rollbar/package.json | 1 - plugins/rollbar/src/components/Router.tsx | 2 +- plugins/scaffolder/package.json | 1 - plugins/scaffolder/src/components/Router.tsx | 2 +- .../ScaffolderPageContextMenu.tsx | 2 +- .../src/components/TaskPage/TaskPage.tsx | 2 +- .../TemplatePage/TemplatePage.test.tsx | 2 +- .../components/TemplatePage/TemplatePage.tsx | 2 +- plugins/scaffolder/src/next/Router/Router.tsx | 2 +- .../TemplateWizardPage/TemplateWizardPage.tsx | 2 +- plugins/search-react/package.json | 2 +- .../DefaultResultListItem.stories.tsx | 2 +- .../SearchResult/SearchResult.stories.tsx | 2 +- plugins/search/package.json | 1 - .../components/SearchPage/SearchPage.test.tsx | 2 +- .../src/components/SearchPage/SearchPage.tsx | 2 +- plugins/sentry/package.json | 2 +- plugins/sentry/src/components/Router.tsx | 2 +- plugins/shortcuts/package.json | 2 +- plugins/shortcuts/src/AddShortcut.tsx | 2 +- plugins/techdocs/package.json | 2 +- .../components/TechDocsIndexPage.test.tsx | 2 +- .../src/home/components/TechDocsIndexPage.tsx | 2 +- .../src/search/components/TechDocsSearch.tsx | 2 +- plugins/todo/package.json | 2 +- plugins/todo/src/plugin.test.tsx | 2 +- plugins/user-settings/package.json | 2 +- .../DefaultSettingsPage.test.tsx | 2 +- .../SettingsPage/SettingsPage.test.tsx | 2 +- .../components/SettingsPage/SettingsPage.tsx | 2 +- 112 files changed, 136 insertions(+), 111 deletions(-) create mode 100644 .changeset/long-eyes-confess.md diff --git a/.changeset/long-eyes-confess.md b/.changeset/long-eyes-confess.md new file mode 100644 index 0000000000..f35504f8d7 --- /dev/null +++ b/.changeset/long-eyes-confess.md @@ -0,0 +1,38 @@ +--- +'@backstage/app-defaults': patch +'@backstage/core-components': patch +'@backstage/dev-utils': patch +'@backstage/test-utils': patch +'@backstage/plugin-airbrake': patch +'@backstage/plugin-api-docs': patch +'@backstage/plugin-azure-devops': patch +'@backstage/plugin-badges': patch +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-graph': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-catalog-react': patch +'@backstage/plugin-circleci': patch +'@backstage/plugin-cloudbuild': patch +'@backstage/plugin-code-climate': patch +'@backstage/plugin-code-coverage': patch +'@backstage/plugin-explore': patch +'@backstage/plugin-git-release-manager': patch +'@backstage/plugin-github-actions': patch +'@backstage/plugin-home': patch +'@backstage/plugin-jenkins': patch +'@backstage/plugin-kafka': patch +'@backstage/plugin-org': patch +'@backstage/plugin-permission-react': patch +'@backstage/plugin-playlist': patch +'@backstage/plugin-rollbar': patch +'@backstage/plugin-scaffolder': patch +'@backstage/plugin-search': patch +'@backstage/plugin-search-react': patch +'@backstage/plugin-sentry': patch +'@backstage/plugin-shortcuts': patch +'@backstage/plugin-techdocs': patch +'@backstage/plugin-todo': patch +'@backstage/plugin-user-settings': patch +--- + +Internal refactor to use `react-router-dom` rather than `react-router`. diff --git a/packages/app-defaults/package.json b/packages/app-defaults/package.json index 5b8e4da681..61eac23cc6 100644 --- a/packages/app-defaults/package.json +++ b/packages/app-defaults/package.json @@ -43,7 +43,6 @@ "peerDependencies": { "react": "^16.13.1 || ^17.0.0", "react-dom": "^16.13.1 || ^17.0.0", - "react-router": "6.0.0-beta.0 || ^6.3.0", "react-router-dom": "6.0.0-beta.0 || ^6.3.0" }, "devDependencies": { diff --git a/packages/app/package.json b/packages/app/package.json index e953838a2f..b7b75f83c2 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -77,7 +77,6 @@ "prop-types": "^15.7.2", "react": "^17.0.2", "react-dom": "^17.0.2", - "react-router": "^6.3.0", "react-router-dom": "^6.3.0", "react-use": "^17.2.4", "zen-observable": "^0.10.0" diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index 05ef61c11a..23bacfceec 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -86,7 +86,7 @@ import { import { AdvancedSettings } from './components/advancedSettings'; import AlarmIcon from '@material-ui/icons/Alarm'; import React from 'react'; -import { Navigate, Route } from 'react-router'; +import { Navigate, Route } from 'react-router-dom'; import { apis } from './apis'; import { entityPage } from './components/catalog/EntityPage'; import { homePage } from './components/home/HomePage'; diff --git a/packages/core-components/package.json b/packages/core-components/package.json index dc2bc08093..4a028e9dc3 100644 --- a/packages/core-components/package.json +++ b/packages/core-components/package.json @@ -74,7 +74,6 @@ "@types/react": "^16.13.1 || ^17.0.0", "react": "^16.13.1 || ^17.0.0", "react-dom": "^16.13.1 || ^17.0.0", - "react-router": "6.0.0-beta.0 || ^6.3.0", "react-router-dom": "6.0.0-beta.0 || ^6.3.0" }, "devDependencies": { diff --git a/packages/core-components/src/components/Button/Button.test.tsx b/packages/core-components/src/components/Button/Button.test.tsx index 805b3b3f10..c5942e3d78 100644 --- a/packages/core-components/src/components/Button/Button.test.tsx +++ b/packages/core-components/src/components/Button/Button.test.tsx @@ -18,7 +18,7 @@ import React from 'react'; import { render, fireEvent, act } from '@testing-library/react'; import { wrapInTestApp } from '@backstage/test-utils'; import { Button } from './Button'; -import { Route, Routes } from 'react-router'; +import { Route, Routes } from 'react-router-dom'; describe('
- + diff --git a/docs/overview/versioning-policy.md b/docs/overview/versioning-policy.md index 10ffc82188..ad5421585e 100644 --- a/docs/overview/versioning-policy.md +++ b/docs/overview/versioning-policy.md @@ -1,7 +1,7 @@ --- id: versioning-policy title: Release & Versioning Policy -description: +description: The process and policy for releasing and versioning Backstage --- The Backstage project is comprised of a set of software components that together diff --git a/docs/releases/v1.2.0-changelog.md b/docs/releases/v1.2.0-changelog.md index 30c780e7ae..977f8a25b7 100644 --- a/docs/releases/v1.2.0-changelog.md +++ b/docs/releases/v1.2.0-changelog.md @@ -722,7 +722,7 @@ ### Patch Changes -- ac19f82936: Added ARIA landmark
to Page component and added ARIA landmark
Sidebar without Catalog - Sidebar with Catalog + Sidebar without CatalogSidebar with Catalog
Before